{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络优化器\n",
    " 待优化参数w，损失函数loss，学习率lr，每次迭代一个batch，t表示当前batch迭代的总参数：  \n",
    "1. 计算t时刻损失函数关于当前参数的梯度$$ g_t =  \\nabla loss = \\frac{\\partial loss}{\\partial w_t} $$\n",
    "2. 计算t时刻一阶动量m_i和二阶动量v_i  \n",
    "3. 计算t时刻下降梯度：$$ \\eta _t =\\frac{lr*m_t}{\\sqrt{V_t}} $$\n",
    "4. 计算t+1时刻参数：$$ w_{t+1} = w_t-\\eta _t = w_t-\\frac{lr*m_t}{\\sqrt{V_t}} $$\n",
    "  \n",
    "一阶动量：与梯度相关的函数  \n",
    "二阶动量：与梯度平方相关的函数  \n",
    "### 随机梯度下降SGD算法\n",
    "$$ m_t = g_t $$\n",
    "$$ V_t = 1 $$\n",
    "$$ \\eta _t =\\frac{lr*m_t}{\\sqrt{V_t}} = lr*g_t $$\n",
    "$$ w_{t+1} = w_t - lr*\\frac{\\partial loss}{\\partial w_t} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, loss: 0.2821310982108116\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 1, loss: 0.25459615513682365\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 2, loss: 0.22570249810814857\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 3, loss: 0.21028399839997292\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 4, loss: 0.19942265003919601\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 5, loss: 0.18873637914657593\n",
      "Test_acc: 0.5\n",
      "--------------------------\n",
      "Epoch 6, loss: 0.17851299047470093\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 7, loss: 0.16922875493764877\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 8, loss: 0.16107673197984695\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 9, loss: 0.15404684841632843\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 10, loss: 0.14802725985646248\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 11, loss: 0.14287303015589714\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 12, loss: 0.1384414155036211\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 13, loss: 0.13460607640445232\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 14, loss: 0.13126072846353054\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 15, loss: 0.12831822223961353\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 16, loss: 0.12570795230567455\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 17, loss: 0.12337299063801765\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 18, loss: 0.12126746587455273\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 19, loss: 0.11935433372855186\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 20, loss: 0.11760355532169342\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 21, loss: 0.11599067971110344\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 22, loss: 0.11449568346142769\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 23, loss: 0.11310207843780518\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 24, loss: 0.11179621890187263\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 25, loss: 0.11056671477854252\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 26, loss: 0.10940408334136009\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 27, loss: 0.10830028541386127\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 28, loss: 0.10724855214357376\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 29, loss: 0.10624313168227673\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 30, loss: 0.1052791029214859\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 31, loss: 0.10435221716761589\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 32, loss: 0.10345886647701263\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 33, loss: 0.10259587876498699\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 34, loss: 0.10176052898168564\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 35, loss: 0.10095042176544666\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 36, loss: 0.10016347281634808\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 37, loss: 0.09939785301685333\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 38, loss: 0.0986519306898117\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 39, loss: 0.09792429022490978\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 40, loss: 0.09721364825963974\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 41, loss: 0.09651889838278294\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 42, loss: 0.09583901055157185\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 43, loss: 0.09517310559749603\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 44, loss: 0.09452036581933498\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 45, loss: 0.0938800759613514\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 46, loss: 0.09325156174600124\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 47, loss: 0.09263424947857857\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 48, loss: 0.09202760085463524\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 49, loss: 0.09143111668527126\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 50, loss: 0.09084436297416687\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 51, loss: 0.09026693925261497\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 52, loss: 0.08969846926629543\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 53, loss: 0.08913860656321049\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 54, loss: 0.08858705498278141\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 55, loss: 0.08804351650178432\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 56, loss: 0.08750772476196289\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 57, loss: 0.0869794450700283\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 58, loss: 0.08645843528211117\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 59, loss: 0.08594449050724506\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 60, loss: 0.08543741703033447\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 61, loss: 0.08493701927363873\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 62, loss: 0.08444313891232014\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 63, loss: 0.08395560272037983\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 64, loss: 0.08347426354885101\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 65, loss: 0.08299898356199265\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 66, loss: 0.08252961561083794\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 67, loss: 0.08206603862345219\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 68, loss: 0.08160812221467495\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 69, loss: 0.08115577697753906\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 70, loss: 0.08070887625217438\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 71, loss: 0.08026730827987194\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 72, loss: 0.07983098551630974\n",
      "Test_acc: 0.6666666666666666\n",
      "--------------------------\n",
      "Epoch 73, loss: 0.07939981296658516\n",
      "Test_acc: 0.6666666666666666\n",
      "--------------------------\n",
      "Epoch 74, loss: 0.0789736919105053\n",
      "Test_acc: 0.6666666666666666\n",
      "--------------------------\n",
      "Epoch 75, loss: 0.07855254225432873\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 76, loss: 0.07813626900315285\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 77, loss: 0.07772481068968773\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 78, loss: 0.07731806673109531\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 79, loss: 0.07691597566008568\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 80, loss: 0.07651844993233681\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 81, loss: 0.07612543925642967\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 82, loss: 0.0757368616759777\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 83, loss: 0.07535265013575554\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 84, loss: 0.07497274875640869\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 85, loss: 0.07459708396345377\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 86, loss: 0.074225596152246\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 87, loss: 0.07385822664946318\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 88, loss: 0.07349492143839598\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 89, loss: 0.07313562091439962\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 90, loss: 0.07278026547282934\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 91, loss: 0.0724287973716855\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 92, loss: 0.07208118494600058\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 93, loss: 0.07173734344542027\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 94, loss: 0.07139724027365446\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 95, loss: 0.07106082234531641\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 96, loss: 0.07072803378105164\n",
      "Test_acc: 0.8\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 97, loss: 0.0703988391906023\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 98, loss: 0.07007317896932364\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 99, loss: 0.0697510140016675\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 100, loss: 0.06943229213356972\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 101, loss: 0.06911696959286928\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 102, loss: 0.06880500447005033\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 103, loss: 0.068496348336339\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 104, loss: 0.06819095648825169\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 105, loss: 0.06788879167288542\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 106, loss: 0.06758981943130493\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 107, loss: 0.0672939820215106\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 108, loss: 0.06700124125927687\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 109, loss: 0.0667115617543459\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 110, loss: 0.06642490904778242\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 111, loss: 0.06614123564213514\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 112, loss: 0.0658605070784688\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 113, loss: 0.06558268051594496\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 114, loss: 0.06530772615224123\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 115, loss: 0.06503560487180948\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 116, loss: 0.06476627103984356\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 117, loss: 0.06449970323592424\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 118, loss: 0.06423585396260023\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 119, loss: 0.06397469434887171\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 120, loss: 0.06371618993580341\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 121, loss: 0.06346031092107296\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 122, loss: 0.06320700887590647\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 123, loss: 0.06295627448707819\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 124, loss: 0.0627080425620079\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 125, loss: 0.06246231310069561\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 126, loss: 0.06221904046833515\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 127, loss: 0.061978183686733246\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 128, loss: 0.06173973251134157\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 129, loss: 0.06150364316999912\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 130, loss: 0.061269884929060936\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 131, loss: 0.06103843171149492\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 132, loss: 0.06080925837159157\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 133, loss: 0.06058232951909304\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 134, loss: 0.06035762187093496\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 135, loss: 0.060135108418762684\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 136, loss: 0.05991474911570549\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 137, loss: 0.05969652533531189\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 138, loss: 0.05948041286319494\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 139, loss: 0.05926638375967741\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 140, loss: 0.05905440915375948\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 141, loss: 0.058844463899731636\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 142, loss: 0.05863652843981981\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 143, loss: 0.058430567383766174\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 144, loss: 0.05822655465453863\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 145, loss: 0.05802448093891144\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 146, loss: 0.057824309915304184\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 147, loss: 0.057626024819910526\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 148, loss: 0.0574295949190855\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 149, loss: 0.0572349950671196\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 150, loss: 0.05704221222549677\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 151, loss: 0.05685121566057205\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 152, loss: 0.05666199512779713\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 153, loss: 0.05647451803088188\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 154, loss: 0.056288767606019974\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 155, loss: 0.05610471125692129\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 156, loss: 0.055922345258295536\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 157, loss: 0.0557416332885623\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 158, loss: 0.05556256230920553\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 159, loss: 0.05538512021303177\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 160, loss: 0.05520927347242832\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 161, loss: 0.0550350034609437\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 162, loss: 0.05486230552196503\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 163, loss: 0.05469114240258932\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 164, loss: 0.05452151130884886\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 165, loss: 0.05435337871313095\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 166, loss: 0.054186731576919556\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 167, loss: 0.054021554067730904\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 168, loss: 0.0538578312844038\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 169, loss: 0.053695546463131905\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 170, loss: 0.0535346744582057\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 171, loss: 0.05337520595639944\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 172, loss: 0.05321711394935846\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 173, loss: 0.05306038819253445\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 174, loss: 0.05290501844137907\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 175, loss: 0.05275098513811827\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 176, loss: 0.0525982566177845\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 177, loss: 0.052446840330958366\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 178, loss: 0.05229670833796263\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 179, loss: 0.05214785039424896\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 180, loss: 0.052000245079398155\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 181, loss: 0.05185388680547476\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 182, loss: 0.05170875135809183\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 183, loss: 0.05156483128666878\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 184, loss: 0.051422109827399254\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 185, loss: 0.05128058139234781\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 186, loss: 0.05114021245390177\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 187, loss: 0.051001012325286865\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 188, loss: 0.05086293909698725\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 189, loss: 0.05072600580751896\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 190, loss: 0.050590199418365955\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 191, loss: 0.050455489195883274\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 192, loss: 0.05032187420874834\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 193, loss: 0.05018933489918709\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 194, loss: 0.050057861022651196\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 195, loss: 0.04992745537310839\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 196, loss: 0.04979808162897825\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 197, loss: 0.04966974165290594\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 198, loss: 0.04954242613166571\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 199, loss: 0.04941611271351576\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 200, loss: 0.04929080419242382\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 201, loss: 0.049166472628712654\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 202, loss: 0.04904312454164028\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 203, loss: 0.048920731991529465\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 204, loss: 0.04879929404705763\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 205, loss: 0.04867880139499903\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 206, loss: 0.04855923820286989\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 207, loss: 0.048440598882734776\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 208, loss: 0.04832287319004536\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 209, loss: 0.048206047154963017\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 210, loss: 0.04809010960161686\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 211, loss: 0.04797505587339401\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 212, loss: 0.047860877588391304\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 213, loss: 0.04774755984544754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 214, loss: 0.0476350924000144\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 215, loss: 0.047523465007543564\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 216, loss: 0.0474126823246479\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 217, loss: 0.04730272479355335\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 218, loss: 0.047193579375743866\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 219, loss: 0.04708524886518717\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 220, loss: 0.04697770718485117\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 221, loss: 0.04687096644192934\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 222, loss: 0.04676500428467989\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 223, loss: 0.046659816056489944\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 224, loss: 0.046555389650166035\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 225, loss: 0.04645173158496618\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 226, loss: 0.046348826959729195\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 227, loss: 0.04624664783477783\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 228, loss: 0.04614521563053131\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 229, loss: 0.04604450520128012\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 230, loss: 0.04594451282173395\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 231, loss: 0.04584523383527994\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 232, loss: 0.045746659860014915\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 233, loss: 0.045648783445358276\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 234, loss: 0.045551598072052\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 235, loss: 0.04545509070158005\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 236, loss: 0.045359269715845585\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 237, loss: 0.0452641025185585\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 238, loss: 0.04516960587352514\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 239, loss: 0.04507576394826174\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 240, loss: 0.04498256929218769\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 241, loss: 0.044890016317367554\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 242, loss: 0.044798108749091625\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 243, loss: 0.04470681492239237\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 244, loss: 0.04461614973843098\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 245, loss: 0.04452611040323973\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 246, loss: 0.04443667363375425\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 247, loss: 0.044347841292619705\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 248, loss: 0.04425961058586836\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 249, loss: 0.044171969406306744\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 250, loss: 0.04408490750938654\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 251, loss: 0.04399843979626894\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 252, loss: 0.043912542052567005\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 253, loss: 0.043827205896377563\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 254, loss: 0.043742443434894085\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 255, loss: 0.043658239766955376\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 256, loss: 0.04357459023594856\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 257, loss: 0.043491488322615623\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 258, loss: 0.04340891819447279\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 259, loss: 0.04332689568400383\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 260, loss: 0.043245404958724976\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 261, loss: 0.043164435774087906\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 262, loss: 0.04308399744331837\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 263, loss: 0.043004066683351994\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 264, loss: 0.04292465094476938\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 265, loss: 0.04284574184566736\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 266, loss: 0.04276733845472336\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 267, loss: 0.042689427733421326\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 268, loss: 0.042612007819116116\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 269, loss: 0.042535082437098026\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 270, loss: 0.04245864413678646\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 271, loss: 0.0423826826736331\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 272, loss: 0.04230719245970249\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 273, loss: 0.042232176288962364\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 274, loss: 0.04215762112289667\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 275, loss: 0.042083531618118286\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 276, loss: 0.04200989846140146\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 277, loss: 0.04193671327084303\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 278, loss: 0.04186398349702358\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 279, loss: 0.04179169982671738\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 280, loss: 0.041719856671988964\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 281, loss: 0.04164844285696745\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 282, loss: 0.0415774704888463\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 283, loss: 0.041506923735141754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 284, loss: 0.041436802595853806\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 285, loss: 0.041367098689079285\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 286, loss: 0.04129782132804394\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 287, loss: 0.04122895281761885\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 288, loss: 0.04116049222648144\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 289, loss: 0.04109244793653488\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 290, loss: 0.04102479945868254\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 291, loss: 0.04095755238085985\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 292, loss: 0.04089069180190563\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 293, loss: 0.04082423448562622\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 294, loss: 0.040758166462183\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 295, loss: 0.040692479349672794\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 296, loss: 0.04062717128545046\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 297, loss: 0.040562248788774014\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 298, loss: 0.040497696958482265\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 299, loss: 0.04043351951986551\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 300, loss: 0.04036971367895603\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 301, loss: 0.04030627105385065\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 302, loss: 0.040243194438517094\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 303, loss: 0.04018046800047159\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 304, loss: 0.04011810338124633\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 305, loss: 0.04005609406158328\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 306, loss: 0.039994440507143736\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 307, loss: 0.03993312222883105\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 308, loss: 0.039872155059129\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 309, loss: 0.03981153108179569\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 310, loss: 0.03975124331191182\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 311, loss: 0.03969129594042897\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 312, loss: 0.039631680119782686\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 313, loss: 0.03957238281145692\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 314, loss: 0.03951342590153217\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 315, loss: 0.03945479029789567\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 316, loss: 0.03939648391678929\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 317, loss: 0.03933848859742284\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 318, loss: 0.03928081085905433\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 319, loss: 0.039223446510732174\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 320, loss: 0.03916640114039183\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 321, loss: 0.03910966124385595\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 322, loss: 0.039053221233189106\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 323, loss: 0.038997095078229904\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 324, loss: 0.03894126741215587\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 325, loss: 0.038885737769305706\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 326, loss: 0.03883049916476011\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 327, loss: 0.038775558583438396\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 328, loss: 0.03872091369703412\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 329, loss: 0.038666554260998964\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 330, loss: 0.0386124849319458\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 331, loss: 0.03855870245024562\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 332, loss: 0.03850520076230168\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 333, loss: 0.03845197660848498\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 334, loss: 0.03839903557673097\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 335, loss: 0.03834636835381389\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 336, loss: 0.03829397959634662\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 337, loss: 0.03824185347184539\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 338, loss: 0.03819000208750367\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 339, loss: 0.038138415198773146\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 340, loss: 0.03808710025623441\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 341, loss: 0.03803604422137141\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 342, loss: 0.03798525780439377\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 343, loss: 0.037934715393930674\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 344, loss: 0.03788443887606263\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 345, loss: 0.03783441847190261\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 346, loss: 0.03778464952483773\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 347, loss: 0.03773513389751315\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 348, loss: 0.03768586507067084\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 349, loss: 0.03763684304431081\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 350, loss: 0.03758806874975562\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 351, loss: 0.03753953706473112\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 352, loss: 0.037491250317543745\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 353, loss: 0.037443204782903194\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 354, loss: 0.03739539487287402\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 355, loss: 0.0373478215187788\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 356, loss: 0.03730047307908535\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 357, loss: 0.03725337516516447\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 358, loss: 0.03720649844035506\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 359, loss: 0.037159846629947424\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 360, loss: 0.03711342951282859\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 361, loss: 0.03706724522635341\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 362, loss: 0.03702127141878009\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 363, loss: 0.036975529976189137\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 364, loss: 0.03693000553175807\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 365, loss: 0.0368847013451159\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 366, loss: 0.036839612759649754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 367, loss: 0.03679474862292409\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 368, loss: 0.03675009310245514\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 369, loss: 0.036705652717500925\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 370, loss: 0.03666142327710986\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 371, loss: 0.036617396865040064\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 372, loss: 0.03657358651980758\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 373, loss: 0.03652997920289636\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 374, loss: 0.036486583296209574\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 375, loss: 0.03644339041784406\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 376, loss: 0.03640039311721921\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 377, loss: 0.0363576035015285\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 378, loss: 0.036315012723207474\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 379, loss: 0.03627262031659484\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 380, loss: 0.036230423022061586\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 381, loss: 0.0361884250305593\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 382, loss: 0.03614661889150739\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 383, loss: 0.03610500367358327\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 384, loss: 0.03606357052922249\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 385, loss: 0.036022343672811985\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 386, loss: 0.035981299821287394\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 387, loss: 0.03594044363126159\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 388, loss: 0.03589977417141199\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 389, loss: 0.035859286319464445\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 390, loss: 0.03581898985430598\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 391, loss: 0.03577886521816254\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 392, loss: 0.03573893057182431\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 393, loss: 0.03569917054846883\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 394, loss: 0.03565958747640252\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 395, loss: 0.0356201883405447\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 396, loss: 0.03558096243068576\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 397, loss: 0.03554190881550312\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 398, loss: 0.03550303215160966\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 399, loss: 0.03546432638540864\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 400, loss: 0.035425789188593626\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 401, loss: 0.03538742894306779\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 402, loss: 0.03534922935068607\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 403, loss: 0.03531120624393225\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 404, loss: 0.035273338202387094\n",
      "Test_acc: 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------\n",
      "Epoch 405, loss: 0.03523564711213112\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 406, loss: 0.03519811388105154\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 407, loss: 0.03516074409708381\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 408, loss: 0.03512354660779238\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 409, loss: 0.035086494870483875\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 410, loss: 0.03504961961880326\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 411, loss: 0.03501288779079914\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 412, loss: 0.03497632360085845\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 413, loss: 0.03493990935385227\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 414, loss: 0.03490366041660309\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 415, loss: 0.03486755723133683\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 416, loss: 0.034831615164875984\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 417, loss: 0.03479582304134965\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 418, loss: 0.03476017666980624\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 419, loss: 0.034724689088761806\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 420, loss: 0.03468935191631317\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 421, loss: 0.03465416422113776\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 422, loss: 0.03461912041530013\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 423, loss: 0.0345842270180583\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 424, loss: 0.03454947331920266\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 425, loss: 0.03451487002894282\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 426, loss: 0.034480408765375614\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 427, loss: 0.03444609511643648\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 428, loss: 0.03441191930323839\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 429, loss: 0.034377886448055506\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 430, loss: 0.0343439974822104\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 431, loss: 0.034310245886445045\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 432, loss: 0.034276632592082024\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 433, loss: 0.03424314968287945\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 434, loss: 0.03420981438830495\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 435, loss: 0.03417661041021347\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 436, loss: 0.03414354659616947\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 437, loss: 0.03411061270162463\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 438, loss: 0.03407781617715955\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 439, loss: 0.03404514491558075\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 440, loss: 0.034012613352388144\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 441, loss: 0.03398020705208182\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 442, loss: 0.03394793486222625\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 443, loss: 0.033915786538273096\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 444, loss: 0.033883772790431976\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 445, loss: 0.03385189129039645\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 446, loss: 0.033820131327956915\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 447, loss: 0.03378849755972624\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 448, loss: 0.03375699184834957\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 449, loss: 0.033725605346262455\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 450, loss: 0.03369434783235192\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 451, loss: 0.03366321325302124\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 452, loss: 0.03363220160827041\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 453, loss: 0.0336013101041317\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 454, loss: 0.03357054525986314\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 455, loss: 0.033539887983351946\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 456, loss: 0.033509367145597935\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 457, loss: 0.03347895201295614\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 458, loss: 0.033448657020926476\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 459, loss: 0.033418480306863785\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 460, loss: 0.03338842652738094\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 461, loss: 0.03335848497226834\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 462, loss: 0.03332865657284856\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 463, loss: 0.033298938535153866\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 464, loss: 0.03326933877542615\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 465, loss: 0.033239856362342834\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 466, loss: 0.03321048244833946\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 467, loss: 0.03318121982738376\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 468, loss: 0.033152075950056314\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 469, loss: 0.03312302986159921\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 470, loss: 0.03309411043301225\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 471, loss: 0.033065286464989185\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 472, loss: 0.03303657751530409\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 473, loss: 0.033007977064698935\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 474, loss: 0.03297947999089956\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 475, loss: 0.03295109001919627\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 476, loss: 0.032922806683927774\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 477, loss: 0.03289463324472308\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 478, loss: 0.03286656038835645\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 479, loss: 0.03283859696239233\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 480, loss: 0.032810729928314686\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 481, loss: 0.03278297185897827\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 482, loss: 0.03275531157851219\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 483, loss: 0.03272775514051318\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 484, loss: 0.03270029602572322\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 485, loss: 0.03267294820398092\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 486, loss: 0.03264569165185094\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 487, loss: 0.03261853335425258\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 488, loss: 0.03259148774668574\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 489, loss: 0.03256453201174736\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 490, loss: 0.032537673600018024\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 491, loss: 0.032510916236788034\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 492, loss: 0.032484252005815506\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 493, loss: 0.032457681372761726\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 494, loss: 0.03243121085688472\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 495, loss: 0.032404832541942596\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 496, loss: 0.032378556206822395\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 497, loss: 0.03235236741602421\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 498, loss: 0.03232627036049962\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 499, loss: 0.032300276681780815\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "total_time 13.468247413635254\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZhcdZ3v8fe3q6u6eu9OdzpLZw+RECCABGQTEFEWnYmOcxVUVAZkUBmHB/SKo5c7zui94rggDl7WCAqIjgMIGlkGhMgAQoJZIYEQEtLphF5Ip/f9e/+o051KdyXpTrpyOlWf1/PUU+f8zjld318eqE/9fqfOKXN3REREhsoJuwARERmfFBAiIpKSAkJERFJSQIiISEoKCBERSUkBISIiKSkgREJgZq1mNifsOkT2RQEhoTGzzWZ2bgive5eZdQdv0gOPT6Tx9Z42s8uT29y9yN03pen1Pmlmy4N+bTezP5jZGel4LclsCgjJVt8L3qQHHr8Ku6CxYGbXADcC/weYBMwAfgosPoC/lTu21cnhRgEh446Z5ZnZjWZWGzxuNLO8YFulmf3OzJrM7B0z+5OZ5QTbvmZm28ysxcw2mNn7R/m6d5nZt5PWzzazmqT1zWb2FTNbbWa7zOxXZhZP2r7YzFaaWbOZvWFm55vZd4D3Av8efKL/92BfN7MjguVSM/u5mdWb2RYz+2ZSnz5nZs+a2ffNbKeZvWlmF+yl/lLgX4AvufsD7t7m7j3u/oi7f3UUffyama0G2oJafjPkdX5sZjcl1X5nMFLZZmbfNrPIaP7dZfzSJwQZj74BnAIcDzjwW+CbwP8CrgVqgInBvqcAbmZHAlcBJ7l7rZnNAtLxRvVx4HygE/hv4HPALWZ2MvBz4G+BJ4EpQLG7P2pmpwP3uPsde/mbPwFKgTlABfA4sB24M9j+HuBuoBK4ArjTzKp9+H1yTgXiwIMH2ceLgQ8BDUAV8E9mVuLuzcGb/8eBjwb73g28DRwBFAK/A7YCtx5kDTIOaAQh49GngH9x9zp3rwe+BVwSbOsh8eY7M/h0/KfgjbIPyAMWmFnU3Te7+xv7eI2vBKOQJjNrGEVtN7l7rbu/AzxCIsQALgOWuPsT7t7v7tvcff3+/ljwhvsJ4Ovu3uLum4EfJPUXYIu73+7ufSTekKeQmD4aqgJocPfeUfQnlZvcfau7d7j7FuBl4CPBtnOAdnd/wcwmARcAVwejlTrgR8BFB/n6Mk4oIGQ8mgpsSVrfErQB/BuwEXjczDaZ2XUA7r4RuBr4Z6DOzO43s6ns3ffdvSx4VI6ith1Jy+1AUbA8HdhXIO1NJRBjeH+rU72mu7cHi0UM1whUjsG5g61D1u8jMaoA+GSwDjATiALbB8KWxMih6iBfX8YJBYSMR7Uk3nwGzAjaCD5lX+vuc4C/Aq4ZONfg7ve5+xnBsQ7cMMrXbQMKktYnj+LYrcDcvWzb1y2TG0iMiob2d9soXnvA8ySmvj6yj31G0seh9f4HcLaZTSMxtTQQEFuBLqAyKWxL3P3oA6hdxiEFhIQtambxpEcu8Evgm2Y20cwqgeuBewDM7MNmdoSZGdBMYmqpz8yONLNzgpPZnUBHsG00VgIXmtkEM5tMYkQyUncCl5rZ+80sx8yqzWx+sO1tEucXhgmmjX4NfMfMis1sJnDNQH9Hw913kfi3utnMPmJmBWYWNbMLzOx7B9rHYJrvaeBnwJvu/mrQvp3E+ZIfmFlJ0O+5ZnbWaGuX8UkBIWFbSuLNfODxz8C3geXAamANiTnwgW/ezAP+C2gl8Yn5p+7+NInzD98l8Yl8B8HJ1VHW8gtgFbCZxBvfiL/66u4vApeSmIPfBTzD7lHBj4G/Db6FdFOKw/+BxCf7TcCzJD6hLxll7QN1/JBEwHwTqCfxKf8q4KFglwPt433AuewePQz4DIkpsleAncBvSJwjkQxg+sEgERFJRSMIERFJSQEhIiIpKSBERCQlBYSIiKSUUbfaqKys9FmzZoVdhojIYWPFihUN7j4x1baMCohZs2axfPnysMsQETlsmNmWvW3TFJOIiKSkgBARkZQUECIiklJGnYMQERmNnp4eampq6OzsDLuUtIvH40ybNo1oNDriYxQQIpK1ampqKC4uZtasWSTu/5iZ3J3GxkZqamqYPXv2iI/TFJOIZK3Ozk4qKioyOhwAzIyKiopRj5QUECKS1TI9HAYcSD8VEMBNT77OM6/Vh12GiMi4ooAAbnnmDf6kgBAR2YMCAojl5tDd1x92GSIi44oCAohFcujuVUCISDhuvfVWvvSlL4VdxjAKCIIRhAJCREKyevVqjj322LDLGEYBQSIgujTFJCIhWbNmzbCAWL9+PWeeeSZHH3005557Lg0NDQDcfffdnHjiiSxcuJD3vve9e20bC7pQDk0xiQh865F1vFLbPKZ/c8HUEv73Xx293/3Wrl3LMcccM7je1dXFxz72Me655x5OOOEEbrjhBn70ox9x3XXXccMNN7By5UpisRhNTU20tLQMaxsrGkEAeZpiEpGQbN26leLiYkpLSwfbHnroIc444wxOOOEEABYsWEBdXR2RSISOjg6uvfZali9fTllZWcq2saIRBDoHISKM6JN+OqQ6//DKK6/s0bZmzRoWLFhAQUEBa9eu5ZFHHuGKK67g8ssv54tf/GLKtrGggCAREJ09CggROfRSnX+orq5m5cqVAGzatIlf/OIXPPvss7z++uvMmzePiy66iFdeeYXOzs6UbWNFAUHiHERzR2/YZYhIFlqzZg2PPvoov/zlLwGYMmUKTz31FEuXLuXYY48lPz+fJUuWUFFRwbXXXsvzzz9PYWEhRx99NLfffjtXXnnlsLaxooBAU0wiEp577703ZftDDz00rO2uu+4aUdtY0UlqIJYb0ZXUIiJDKCDQ11xFRFJRQBBcKKeAEMlK7h52CYfEgfRTAcHAdRB9YZchIodYPB6nsbEx40Ni4Bfl4vH4qI7TSWp0N1eRbDVt2jRqamqor8/82/0P/Cb1aCgg0DkIkWwVjUZH9RvN2UZTTCRGEP0OvRpFiIgMUkCQCAhA00wiIkkUECSmmABNM4mIJFFAkDSCUECIiAxSQLB7BKFrIUREdlNAoHMQIiKppDUgzOx8M9tgZhvN7LoU2z9lZquDx3NmdlzSts1mtsbMVprZ8nTWqSkmEZHh0nYdhJlFgJuBDwA1wEtm9rC7v5K025vAWe6+08wuAG4D3pO0/X3u3pCuGgfoJLWIyHDpHEGcDGx0903u3g3cDyxO3sHdn3P3ncHqC8DoLvMbI5piEhEZLp0BUQ1sTVqvCdr25jLgD0nrDjxuZivM7Iq9HWRmV5jZcjNbfqCXy2uKSURkuHTeasNStKW8I5aZvY9EQJyR1Hy6u9eaWRXwhJmtd/dlw/6g+20kpqZYtGjRAd1xSwEhIjJcOkcQNcD0pPVpQO3QncxsIXAHsNjdGwfa3b02eK4DHiQxZZUW+pqriMhw6QyIl4B5ZjbbzGLARcDDyTuY2QzgAeASd38tqb3QzIoHloEPAmvTVWiezkGIiAyTtikmd+81s6uAx4AIsMTd15nZlcH2W4DrgQrgp2YG0Ovui4BJwINBWy5wn7s/mq5aNcUkIjJcWm/37e5LgaVD2m5JWr4cuDzFcZuA44a2p4sCQkRkOF1JTfJ1EPpVORGRAQoIdB2EiEgqCgg0xSQikooCAt1qQ0QkFQUEYGbEIjl0aYpJRGSQAiIQy83RCEJEJIkCIqCAEBHZkwIiEIsoIEREkikgArHcHH3NVUQkiQIioCkmEZE9KSACsUgOPRpBiIgMUkAEYrk5ut23iEgSBURAASEisicFRKAgFqGzRzfrExEZoIAIFMZyae3qDbsMEZFxQwERKMyL0KaAEBEZpIAIFObl0t6lKSYRkQEKiEBRXi5t3b24e9iliIiMCwqIQGFeLv0OHTpRLSICKCAGFeYlfp5bJ6pFRBIUEIHCWASANp2HEBEBFBCDBkYQ+iaTiEiCAiJQpIAQEdmDAiIwOILoVkCIiIACYlBRXuIcRKvOQYiIAAqIQcXxKADNHT0hVyIiMj4oIAKl+YmA2KWAEBEBFBCD4tEI+dEIO9u6wy5FRGRcUEAkKS+I0qQRhIgIoIDYQ1lBjKZ2jSBERCDNAWFm55vZBjPbaGbXpdj+KTNbHTyeM7PjRnpsOpQVRGlq1whCRATSGBBmFgFuBi4AFgAXm9mCIbu9CZzl7guBfwVuG8WxY668IMZOjSBERID0jiBOBja6+yZ37wbuBxYn7+Duz7n7zmD1BWDaSI9Nh1KNIEREBqUzIKqBrUnrNUHb3lwG/GG0x5rZFWa23MyW19fXH0S5u09S6zchRETSGxCWoi3lO6+ZvY9EQHxttMe6+23uvsjdF02cOPGACh1QXhCjr99p0f2YRETSGhA1wPSk9WlA7dCdzGwhcAew2N0bR3PsWBu4WK6pTdNMIiLpDIiXgHlmNtvMYsBFwMPJO5jZDOAB4BJ3f200x6ZDeUEMgKYOnagWEclN1x92914zuwp4DIgAS9x9nZldGWy/BbgeqAB+amYAvcF0Ucpj01XrgPLCxAhip05Ui4ikLyAA3H0psHRI2y1Jy5cDl4/02HQrzQ9GEPqqq4iIrqROVl4QnIPQCEJERAGRbOAktS6WExFRQOwhN5JDcTxXIwgRERQQw5Trhn0iIoACYpjygqi+xSQiggJimNKCmH4TQkQEBcQw5QVRTTGJiKCAGCZxDkIjCBERBcQQpflRmjt76OvXHV1FJLspIIYoL4jiDrt0HkJEspwCYoiyAt1uQ0QEFBDDlBcmAuKdNgWEiGQ3BcQQlUWJgGhoVUCISHZTQAxRWZQHQENrV8iViIiESwExxIRgiqlRIwgRyXIKiCGikRzKCqIaQYhI1lNApFBZlKeAEJGsp4BIobIopikmEcl6CogUKjSCEBFRQKQyUQEhIjKygDCzQjPLCZbfZWZ/bWbR9JYWnorCGM2dvXT19oVdiohIaEY6glgGxM2sGngSuBS4K11Fha2yOHEthM5DiEg2G2lAmLu3A38D/MTdPwosSF9Z4arQtRAiIiMPCDM7FfgU8PugLTc9JYVvYASh8xAiks1GGhBXA18HHnT3dWY2B/hj+soK18Tgdhv1CggRyWIjGgW4+zPAMwDByeoGd/9yOgsL08RgBFHX3BlyJSIi4Rnpt5juM7MSMysEXgE2mNlX01taeOLRCJVFMbY1dYRdiohIaEY6xbTA3ZuBjwBLgRnAJWmrahyYWpbPtiaNIEQke400IKLBdQ8fAX7r7j1ARv9o89TSfGo1ghCRLDbSgLgV2AwUAsvMbCbQnK6ixoOpZfls29mBe0bnoIjIXo0oINz9JnevdvcLPWEL8L401xaq6vJ8Onr6aGrvCbsUEZFQjPQkdamZ/dDMlgePH5AYTezvuPPNbIOZbTSz61Jsn29mz5tZl5l9Zci2zWa2xsxWmtnyEfdojFSXxQF0olpEstZIp5iWAC3Ax4NHM/CzfR1gZhHgZuACElddX2xmQ6++fgf4MvD9vfyZ97n78e6+aIR1jpmpZfkAOg8hIllrpFdDz3X3jyWtf8vMVu7nmJOBje6+CcDM7gcWk/iaLADuXgfUmdmHRlHzITEQEBpBiEi2GukIosPMzhhYMbPTgf29c1YDW5PWa4K2kXLgcTNbYWZX7G0nM7tiYOqrvr5+FH9+3yoKY+Tl5mgEISJZa6QjiCuBn5tZabC+E/jsfo6xFG2j+UrQ6e5ea2ZVwBNmtt7dlw37g+63AbcBLFq0aMy+cmRmVJfn89Y77WP1J0VEDisj/RbTKnc/DlgILHT3E4Bz9nNYDTA9aX0aUDvSwty9NniuAx4kMWV1SM2dWMTGutZD/bIiIuPCqH5Rzt2bgyuqAa7Zz+4vAfPMbLaZxYCLgIdH8jrBDxQVDywDHwTWjqbWsXBEVRFbGtvp6es/1C8tIhK6g7lld6oppEHu3mtmVwGPARFgSXAn2CuD7beY2WRgOVAC9JvZ1SS+8VQJPGhmAzXe5+6PHkStB2ReVRG9/c6WxjaOqCo+1C8vIhKqgwmI/c73u/tSEvduSm67JWl5B4mpp6GageMOorYxcURVEQAb61oVECKSdfYZEGbWQuogMCA/LRWNI3Mn7g4IEZFss8+AcPes/thcmJfL1NK4AkJEstKoTlJno7lVRbyugBCRLKSA2I8FU0p47e0WOnv6wi5FROSQUkDsxwkzyujpc17ZntF3NxcRGUYBsR/HTy8HYOVbTSFXIiJyaCkg9mNyaZzJJXFW1SggRCS7KCBG4PjpZazcqoAQkeyigBiB42eUsaWxnfqWrrBLERE5ZBQQI3Da3AoA/ntjQ8iViIgcOgqIEThmaikTCmM889rY/d6EiMh4p4AYgZwc44wjKvnT6/X094/ZT06IiIxrCogROvNdE2lo7db1ECKSNRQQI3T2kRPJMXh07Y6wSxEROSQUECNUWZTHaXMreXhVLe6aZhKRzKeAGIW/Pm4qb73TzqqaXWGXIiKSdgqIUTjvmMnEIjk88HJN2KWIiKSdAmIUSvOjfHjhFP5zRQ0tnT1hlyMiklYKiFG65NSZtHX38eBftoVdiohIWikgRun46WUcN62UO599k96+/rDLERFJGwXEKJkZXzj7CLY0tvPwqtqwyxERSRsFxAH44IJJzJ9czE+e2kiPRhEikqEUEAcgJ8f46nlH8mZDG3c/tznsckRE0kIBcYDOmV/F2UdO5Mb/ep26ls6wyxERGXMKiANkZlz/4QV09fbx3aXrwy5HRGTMKSAOwpyJRVx51lwe+Ms2HlunezSJSGZRQBykfzhnHsdUl/D1B9ZoqklEMooC4iDFcnP40cePp62rl2t+tUrXRohIxlBAjIF5k4r518XH8OzGBv7t8Q1hlyMiMiZywy4gU3z8pOms3tbErc9sYv7kYj56wrSwSxIROShpHUGY2flmtsHMNprZdSm2zzez582sy8y+Mppjx6PrP3w0p8yZwFf/YzXL9PvVInKYS1tAmFkEuBm4AFgAXGxmC4bs9g7wZeD7B3DsuBPLzeG2zyxi3qRirrxnBau2NoVdkojIAUvnCOJkYKO7b3L3buB+YHHyDu5e5+4vAUPvnb3fY8erkniUuy89iQmFMT77sxdZu00/LiQih6d0BkQ1sDVpvSZoG9NjzewKM1tuZsvr68fHtE5VSZz7Lj+FwlguF9/+An95a2fYJYmIjFo6A8JStI30x5xHfKy73+bui9x90cSJE0dcXLrNqCjgV39/ChMKY3z6jj/z4pvvhF2SiMiopDMgaoDpSevTgJHeH/tgjh03ppUX8KsrTmVSaZxL7vwzj67V1dYicvhIZ0C8BMwzs9lmFgMuAh4+BMeOK5NL4/z670/lqCklfOHeFdzxp024j3QgJSISnrQFhLv3AlcBjwGvAr9293VmdqWZXQlgZpPNrAa4BvimmdWYWcnejk1XrelWWZTHLz9/CuctmMy3f/8q1/92nX5HQkTGPcukT7OLFi3y5cuXh13GXvX3O999dD23LdvESbPKufmT76aqJB52WSKSxcxshbsvSrVNt9o4hHJyjH+68Ch+fNHxrN3WzId+8iwvbdbJaxEZnxQQIVh8fDUPfel0ivJyufi2F7h92Sb6+zNnJCcimUEBEZIjJxfz26tO5/1HVfGdpa9yyZI/s2OXbhcuIuOHAiJEJfEot3z6RP7v3xzLy1uaOO/GZSxdsz3sskREAAVE6MyMi0+ewdJ/fC+zKgr44r0vc82vV9LU3h12aSKS5RQQ48TsykJ+84XT+IdzjuC3K2s594fL+P3q7bpmQkRCo4AYR6KRHK794JE8fNXpTCmN86X7XubzP1/B9l0dYZcmIllIATEOHT21lAe/eBrfuPAont1Yzwd+uIzbl22iu1cX14nIoaOAGKdyIzl8/sw5PH71WZw0q5zvLH2V83+8jKc31IVdmohkCQXEODejooCfXXoySz63CHf43M9e4rK7XmJzQ1vYpYlIhlNAHCbOmT+Jx64+k69fMJ8XNjXygR89wz8/vI76lq6wSxORDKV7MR2G6po7ufHJ1/nVS1vJy83hsjNm8/kz51ASj4ZdmogcZvZ1LyYFxGHszYY2fvD4Bn63ejtlBVG+cNZcPn3KTArzcsMuTUQOEwqIDLd22y6+99gGlr1WT3lBlL87fTafOW0WpfkaUYjIvikgssSKLTu5+Y8beWp9HcV5uXzmtJn83emzqSjKC7s0ERmnFBBZZl3tLn76xzdYunY78dwI/2PRND532izmTCwKuzQRGWcUEFlqY10LtzyziYdX1tLd188586u49PRZnHFEJWYWdnkiMg4oILJcfUsX9/55C/e8sIWG1m7eNamIz542i8XHV1OkE9oiWU0BIQB09fbxyKrt/Oy/32RdbTMFsQh/tXAqF508neOnl2lUIZKFFBCyB3fnL1ubuP/Ft3hk1XY6evqYP7mYi06azkdPmEZpgb79JJItFBCyVy2dPTyyajv3v/QWq2t2EcvN4f3zq1h8/FTOPrKKeDQSdokikkYKCBmRdbW7+M2KGh5ZtZ2G1i6K47lceMwUFp8wlVNmV5CToykokUyjgJBR6e3r57k3Gnlo5TYeW7uDtu4+JpXkcf7Rkznv6MmcPHsCuRHdxkskEygg5IB1dPfxX6++zSOraln2ej2dPf2UF0R5/1GTOP/oyZwxr1LTUCKHMQWEjIn27l6WvVbPo2t38OT6Olo6eymMRTjzXRM5+8iJnPWuKiaXxsMuU0RGYV8BoS/By4gVxHI5/5gpnH/MFLp7+3l+UyOPrt3BU+vf5g9rdwAwf3IxZx9ZxdlHTuTEmeVENRUlctjSCEIOmruzfkcLz7xWz9Mb6li+eSe9/U5xXi6nHVHBaXMrOXVuBfOqinSthcg4oykmOaRaOnt47o1Gnt5Qz7LX6tnW1AFAZVGM98yp4NQ5FZw6t4I5lYUKDJGQaYpJDqnieJTzgm88AWx9p53n32jk+U2NPP9GI79fvR2ASSV5LJo1gRNnlHPizHIWTC3RlJTIOKKAkLSbPqGA6RMK+PhJ03F3NjcmAuOFTY2s2LJzMDDi0RwWTivjxJnlnDijnHfPLGdCYSzk6kWyl6aYJHTbd3Xw8pYmVmzZyYq3drJu2y56+xP/XU6fkM+x1aUcU13KscGjrEChITJWQptiMrPzgR8DEeAOd//ukO0WbL8QaAc+5+4vB9s2Ay1AH9C7tw7I4W9KaT4fWpjPhxZOAaCzp4/VNbtYsWUna7ftYvW2Jpau2TG4/9DQWDClRD+KJJIGaQsIM4sANwMfAGqAl8zsYXd/JWm3C4B5weM9wP8Lnge8z90b0lWjjE/xaISTZ0/g5NkTBtua2rtZu62ZNdt2sXbbLtZs27VHaFQW5TF/cjFHBo/5k4uZV1VMfkwX8YkcqHSOIE4GNrr7JgAzux9YDCQHxGLg556Y53rBzMrMbIq7b09jXXIYKiuIcca8Ss6YVznYNhAa63c0s35HCxt2tHDPC1vo6u0HwAxmVRRy5KREaMytKmLuxELmVBYpOERGIJ0BUQ1sTVqvYc/Rwd72qQa2Aw48bmYO3Orut6V6ETO7ArgCYMaMGWNTuRwWUoVGX7+zpbGNDTtaBkNjw9stPPbKDpJPt1WX5TNnYiFzKguZW1XEnMoi5lYVMrkkrq/eigTSGRCp/i8bekZ8X/uc7u61ZlYFPGFm69192bCdE8FxGyROUh9MwXL4i+QYcyYWMWdiERccO2WwvbOnjzcb2thU38am+lbeqG9lU0Mbv1lRQ1t33+B+BbEIMysKmTmhgBkVBcyYkHjMrChgalm+voYrWSWdAVEDTE9anwbUjnQfdx94rjOzB0lMWQ0LCJGRiEcjHDWlhKOmlOzR7u7UtXTxRn0rbwThsaWxndfrWnhqQx3dwXQVQI7B1LJ8Zg4GRyEzJhRQXZ7P1LI4lYV5uiW6ZJR0BsRLwDwzmw1sAy4CPjlkn4eBq4LzE+8Bdrn7djMrBHLcvSVY/iDwL2msVbKUmTGpJM6kkjinza3cY1t/v/N2SydvNbbz1ju7H1sa23l83ds0tnXvsX8sksOUsjhTS/OZWpZPdVmcqWX5wSOxXBDTpUdy+Ejbf63u3mtmVwGPkfia6xJ3X2dmVwbbbwGWkviK60YSX3O9NDh8EvBgMBecC9zn7o+mq1aRVHJyjCml+Uwpzec9cyqGbW/p7GHrOx3UNnVQu6uDbU0d1DZ1UtvUwfNvNLCjuZP+IZOeZQVRppTmM7kkj6riOJNK8qgKAmpSSR6TSuJUFMb0exsyLuhCOZE06e3r5+2WrkSANA0ESCJE6lo6ebu5i8bWrmEhkmNQUZSXCIzieBAgiUCpKs6joihGZVHiWSMSOVi6F5NICHIjOVSX5VNdlr/XfXr7+mls6+bt5kRgvN3cSV1zJ3UtieXtuzpZVdNEQ2t3yuPzo5HBwKgsilFRmAiOiqT1yuLEc3lBVCMTGRUFhEiIciM5g+dA9qWnr5/6li4aWrtobO1OPLd109ASPLd2UdvUyZptu2hs7R68VUkyMyjNj1JeEAueE8tlBTHKC6KUFcaS2hLP5QUxXTOSxRQQIoeBaCRn8IT3/vT3O82dPTS0dtPYujtAGlq72dnWzc72bprae6hr6eK1t1tpau/e46u+Q+Xl5gyGRllSqJTk51ISj1KSH6Uknhs8RylNas/LzdF1JYcxBYRIhsnJMcqCN/EjqopGdExXbx+72nvY2d4TBEh30nLP4HpTezev17XS1N5DS2fP4FXrexOL5OwZJEPCJHlbcV4uRfFcCmO5FMdzKczLpSgvl1iupsXCooAQEfJyI1SVRKjaz1TXUJ09fTR39tDc0Rs899Dc2Rs8p26v2dmeaO/oobtv3wEDiZApzIukDI+BR2FeivZg/8T2CAWxXOJRjWhGQwEhIgcsHo0Qj0aoKj6w43cHTA8tnb20dfXR2tVDa1cfrZ09tHX3Be29tA48Ont5p62btxrbB9va9zFFlswMCqIR8mOJ0MiPRiiIRSjMyx1cLsjLpSB5OZbYrzAvl/xYhILk5VgieApikYy8yl4BISKh2R0wo9/4ogQAAAYpSURBVBu5DNXX77R1B0HSuTtM2rp6BwOmvaePju4+2rv7aO/uDZ4Tba1dvdS3dA3bNhrRiAUhkxipDPQtHs0hP1jOj0bIC54H9kle3r0+pC0WIZ6bEzxHDtkV+woIETnsRXIscS4jHoXSsfmb7k5nTz9t3b2DwZK8vGfI9NIWhE1Hdx+dvQPP/XR299HY1p3U3k9XTx8dPX0pv202ErFIzu6AiUWYVBzn11eeOjYdT6KAEBFJwczIj0XS+jXf3r5+Onv7E+HRM/DopyNY7ujZs313W/9ge0dPH/nR9NSogBARCUluJIeiSA5FeePzrTjzzqqIiMiYUECIiEhKCggREUlJASEiIikpIEREJCUFhIiIpKSAEBGRlBQQIiKSUkb95KiZ1QNbDvDwSqBhDMs5HKjP2UF9zg4H2ueZ7j4x1YaMCoiDYWbL9/a7rJlKfc4O6nN2SEefNcUkIiIpKSBERCQlBcRut4VdQAjU5+ygPmeHMe+zzkGIiEhKGkGIiEhKCggREUkp6wPCzM43sw1mttHMrgu7nrFiZkvMrM7M1ia1TTCzJ8zs9eC5PGnb14N/gw1mdl44VR8cM5tuZn80s1fNbJ2Z/WPQnrH9NrO4mb1oZquCPn8raM/YPg8ws4iZ/cXMfhesZ3SfzWyzma0xs5VmtjxoS2+f3T1rH0AEeAOYA8SAVcCCsOsao76dCbwbWJvU9j3gumD5OuCGYHlB0Pc8YHbwbxIJuw8H0OcpwLuD5WLgtaBvGdtvwICiYDkK/Bk4JZP7nNT3a4D7gN8F6xndZ2AzUDmkLa19zvYRxMnARnff5O7dwP3A4pBrGhPuvgx4Z0jzYuDuYPlu4CNJ7fe7e5e7vwlsJPFvc1hx9+3u/nKw3AK8ClSTwf32hNZgNRo8nAzuM4CZTQM+BNyR1JzRfd6LtPY52wOiGtiatF4TtGWqSe6+HRJvpkBV0J5x/w5mNgs4gcQn6ozudzDVshKoA55w94zvM3Aj8D+B/qS2TO+zA4+b2QozuyJoS2ufx+cvZR86lqItG7/3m1H/DmZWBPwncLW7N5ul6l5i1xRth12/3b0PON7MyoAHzeyYfex+2PfZzD4M1Ln7CjM7eySHpGg7rPocON3da82sCnjCzNbvY98x6XO2jyBqgOlJ69OA2pBqORTeNrMpAMFzXdCeMf8OZhYlEQ73uvsDQXPG9xvA3ZuAp4Hzyew+nw78tZltJjEtfI6Z3UNm9xl3rw2e64AHSUwZpbXP2R4QLwHzzGy2mcWAi4CHQ64pnR4GPhssfxb4bVL7RWaWZ2azgXnAiyHUd1AsMVS4E3jV3X+YtClj+21mE4ORA2aWD5wLrCeD++zuX3f3ae4+i8T/s0+5+6fJ4D6bWaGZFQ8sAx8E1pLuPod9Zj7sB3AhiW+7vAF8I+x6xrBfvwS2Az0kPk1cBlQATwKvB88Tkvb/RvBvsAG4IOz6D7DPZ5AYRq8GVgaPCzO538BC4C9Bn9cC1wftGdvnIf0/m93fYsrYPpP4puWq4LFu4L0q3X3WrTZERCSlbJ9iEhGRvVBAiIhISgoIERFJSQEhIiIpKSBERCQlBYTIKJhZX3A3zYHHmN0B2MxmJd99VyRs2X6rDZHR6nD348MuQuRQ0AhCZAwE9+q/IfhthhfN7IigfaaZPWlmq4PnGUH7JDN7MPgdh1VmdlrwpyJmdnvw2w6PB1dHi4RCASEyOvlDppg+kbSt2d1PBv6dxN1GCZZ/7u4LgXuBm4L2m4Bn3P04Er/bsS5onwfc7O5HA03Ax9LcH5G90pXUIqNgZq3uXpSifTNwjrtvCm4YuMPdK8ysAZji7j1B+3Z3rzSzemCau3cl/Y1ZJG7XPS9Y/xoQdfdvp79nIsNpBCEydnwvy3vbJ5WupOU+dJ5QQqSAEBk7n0h6fj5Yfo7EHUcBPgU8Gyw/CXwBBn/wp+RQFSkyUvp0IjI6+cGvtw141N0HvuqaZ2Z/JvHB6+Kg7cvAEjP7KlAPXBq0/yNwm5ldRmKk8AUSd98VGTd0DkJkDATnIBa5e0PYtYiMFU0xiYhIShpBiIhIShpBiIhISgoIERFJSQEhIiIpKSBERCQlBYSIiKT0/wHjSkvN283X+AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAd1ElEQVR4nO3de5xcZZ3n8c83nU7SuUBIAgHSSTpCmFyGi9AEXggMGrk5KLIDArprcFWIC95GZ0FlZVhXR0dxlAWGCSwD7KoRRFARuSw3lYshQIA0AXIhhCYEQrrDpbuT7nT/5o861V00laST9OmqrvN9v1716jpPnTr1PA3pbz3nec5zFBGYmVl2DSl1BczMrLQcBGZmGecgMDPLOAeBmVnGOQjMzDLOQWBmlnEOAjOzjHMQWEWT9ICkZknDUzi2JH1J0lJJLZIaJd0s6cD+/iyzNDkIrGJJqgOOAQL4WAof8VPgy8CXgHHAAcBtwN/u6IEkDe3fqpn1nYPAKtmngUeB64F5hS9Imizp15LWS9og6YqC1z4vaZmktyU9K+nQ3geWNB04Hzg7Iu6LiM0R0RoRP4uI7yf7PCDpcwXvOUfSnwu2Q9L5kpYDyyVdLelHvT7nN5L+Pnm+r6Rbkjq/KOlL/fA7MnMQWEX7NPCz5HGipIkAkqqA24GXgDpgErAwee0M4B+T9+5Griexocix5wKNEbFoF+v4ceAIYBbwc+BMSUrqsgdwArBQ0hDgd8BTSX3nAl+RdOIufr6Zg8Aqk6SjganATRHxOLAS+GTy8hxgX+AfIqIlIjZFRP6b+ueAf46IxyJnRUS8VOQjxgOv9kNV/ykimiKiDfgTudNYxySvnQ48EhFrgcOBPSPif0ZEe0SsAq4BzuqHOljGOQisUs0D7o6IN5Ltn9Nzemgy8FJEbCnyvsnkQmN7NgD77HIt4eX8k8itALkQODsp+iS53gzkQm1fSRvzD+CbwMR+qINlnAeorOJIqgE+AVRJWpcUDwfGSjqY3B/fKZKGFgmDl4H9+vAx9wJXSqqPiMVb2acFGFmwvXeRfXov//sL4G5J3yd3yui0gnq9GBHT+1A3sx3iHoFVoo8DneTOux+SPGaSO/XyaWARudM635c0StIISR9I3nst8HVJhyXTQ/eXNLX3B0TEcuAq4BeSjpM0LDnOWZIuSnZbAvwnSSMl7Q98dnsVj4gngfVJPe6KiI3JS4uAtyRdKKlGUpWkv5Z0+M78gswKOQisEs0D/j0i1kTEuvwDuAL4FCDgo8D+wBqgETgTICJuBr5L7lTS2+Smg47byud8KTnmlcBGcqeUTiM3qAvwL0A78BpwAz2nebbnF8CHkzqQ1KszqfMhwIvAG+TCYvc+HtNsq+Qb05iZZZt7BGZmGecgMDPLOAeBmVnGOQjMzDJu0F1HMGHChKirqyt1NczMBpXHH3/8jYjYs9hrgy4I6urqWLx4a9fvmJlZMZKKLZUC+NSQmVnmOQjMzDLOQWBmlnEOAjOzjHMQmJllXGpBIOk6Sa9LWrqV1yXpckkrJD1d7HaAZmaWvjR7BNcDJ23j9ZOB6cnjXOBfU6yLmZltRWrXEUTEHyXVbWOXU4Ebk7syPSpprKR9IqI/bv9nFWjtxjZuWvwyXV1eMdeyqb5uHMceUPSasF1SygvKJlFwmz5ya8JPosh9YCWdS67XwJQpUwakclZ+Fi5aw+X3rSB3a3ez7Jn/N/tVXBAU++dc9KteRCwAFgDU19f762BGvdHSzvhRw3j8fxxf6qqYVZRSzhpqJHej8LxaYG2J6mKDwMbWdvYYNazU1TCrOKUMgt8Cn05mDx0JvOnxAduWppZ2xo10EJj1t9RODUn6BXAcMEFSI3AJUA0QEVcDdwAfAVYArcBn0qqLVYbmlg7qJowsdTXMKk6as4bO3s7rAZyf1udb5WlqbefQUWNLXQ2ziuMri21QiAiaW9rZw6eGzPrdoLsfgVWGN1s7+Om9y2nr6OzT/p1dXWzpCgeBWQocBFYSD618g+seepFxo4ZRNaRvFwZMGlvDoVN9asisvzkIrCSaWtoBuPPLx7DXbiNKXBuzbPMYgZVEcxIEY32qx6zkHARWEk2t7YwZPpRhQ/2/oFmp+V+hlURzi68SNisXDgIriabWDgeBWZlwEFhJ5K4JqC51NcwMzxqyxGV3P8+yV98esM9buf4dpu+194B9npltnYPA2NLZxRX3r2DP0cOZMHr4gHzmtAmjOH7WxAH5LDPbNgeB8WZbBxFw/gf3Z95RdaWujpkNMI8RGM2tuTn9Hrw1yyYHgdHU0gHgtf7NMspBYN3LPewxyrN4zLLIQWA9p4bcIzDLJAeBOQjMMs6zhjKksyu44OdPsPbNTe8qf3VjGzXVVdQMqypRzcyslBwEGfLaW5v4w9J1zNh7DBMLln4eW1PNIZO9zr9ZVjkIMiQ/KPzV4w/gxNm+qtfMcjxGkCEeCzCzYhwEGdLcmlwv4GmiZlbAQZAh+buCuUdgZoUcBBnS1NKOBLvXuEdgZj0cBBmysbWd3UZUM7TK/9nNrIdnDVWw+f/3cZ555c3u7aaWdibuNjDLTJvZ4OEgqFCdXcFdz65j1j67MXOf3brLj95/QglrZWblyEFQofL3GDjjsFrO+cC0UlfHzMqYTxZXqJ4VRT1DyMy2zUFQoTb64jEz6yMHQYXK9wjGuUdgZtvhIKhQvv2kmfWVg6BC5ZeT2GOkLx4zs21zEFSo5pZ2hg8dQk217zFgZtvmIKhQTS3tjBs1DEmlroqZlTkHQYVqbm1nrGcMmVkfOAgqVHNrh5ebNrM+cRBUqOaWdl9DYGZ94iCoUE2tDgIz65tUg0DSSZKel7RC0kVFXt9d0u8kPSWpQdJn0qxPVnR2BW+2dfgaAjPrk9SCQFIVcCVwMjALOFvSrF67nQ88GxEHA8cBl0nyX6+dFJELgMbmViJgnK8hMLM+SHP10TnAiohYBSBpIXAq8GzBPgGMUW6O42igCdiSYp0q2k/+/3J+eu/y7u3xo33vATPbvjSDYBLwcsF2I3BEr32uAH4LrAXGAGdGRFfvA0k6FzgXYMqUKalUthI8v+5tJu42nPOO3Y/h1UP48MyJpa6SmQ0CaQZBsSuZotf2icAS4EPAfsA9kv4UEW+9600RC4AFAPX19b2PYYnm1namjh/Ffz3a9x8ws75Lc7C4EZhcsF1L7pt/oc8Av46cFcCLwIwU61TRmlvbGeeZQma2g9IMgseA6ZKmJQPAZ5E7DVRoDTAXQNJE4K+AVSnWqaI1tXimkJntuNRODUXEFkkXAHcBVcB1EdEgaX7y+tXAd4DrJT1D7lTShRHxRlp1qmQRwcbWdq82amY7LNV7FkfEHcAdvcquLni+FjghzTpkxdubt7ClK3wjGjPbYb55/SAVEaze0EpnV27sfN2bmwDfmtLMdpyDYJC64eHV/OPvnn1P+V67+doBM9sxDoJB6qWmVmqqq/jB6Qd1l42sruKo/SaUsFZmNhg5CAapja0dTBgzjI8dvG+pq2Jmg5xXHx2kmlp8zYCZ9Q8HwSDlO5CZWX9xEAxSza3tnipqZv3CQTBINbd0eKqomfULDxYPMi83tbKmqZV3Nm/xPYnNrF84CAaZ069+mNfe2gzAPrvXlLg2ZlYJHASDyJbOLl57azOnH1bL2XMmc3Dt2FJXycwqgINgENnY1gHAgZN257Cp40pcGzOrFB4sHkSaW9oBvNS0mfUrB8Eg0pQEgS8kM7P+5CAYRJpbc6eG9vBsITPrRw6CQaS5NTk15B6BmfUjDxaXuQ3vbObhlRsI4NFVGwAHgZn1LwdBmfvf963g+odXd2/vNWY4NcOqSlchM6s4DoIy99pbm6gbP5Jr5x0OwJ6jfeMZM+tfDoIy19TSzl67jWD/vUaXuipmVqE8WFzmmlt93wEzS5eDoMw1tXT4AjIzS5WDoIxFRHLfAV83YGbpcRCUsbc2baGzKzxd1MxS5SAoU2+2dXBjMm3UQWBmaXIQlKnbnnyFy+55gaoh8owhM0uVp4+WqTfe2cwQwdOXnMCo4f7PZGbpcY+gTDW1tLPHyGEOATNLnYOgTDW3tjN2pGcLmVn6HARlqrmlg3G+fsDMBoCDoEw1t7Z7tpCZDQgHQZnKjxGYmaXNI5El9tKGFn752Mt0xbvLm1ravbSEmQ0IB0GJ/XzRGv7twVUMG/ruzll11RAOmbx7iWplZlniICixpnfa2Wf3ETzyjbmlroqZZZTHCErMg8JmVmoOghJramn3NFEzKykHQYk1t3b4wjEzK6lUg0DSSZKel7RC0kVb2ec4SUskNUh6MM36lKPc/QbcIzCz0tnuYLGkUUBbRHQl20OAERHRup33VQFXAscDjcBjkn4bEc8W7DMWuAo4KSLWSNpr55sy+Gzp7OLNtg6PEZhZSfVl1tC9wIeBd5LtkcDdwFHbed8cYEVErAKQtBA4FXi2YJ9PAr+OiDUAEfF636s++Fx5/wpWrW/p3u7o7CIC9wjMrKT6EgQjIiIfAkTEO5JG9uF9k4CXC7YbgSN67XMAUC3pAWAM8NOIuLH3gSSdC5wLMGXKlD58dPlpa+/kh3c9z+411YwuWFH0fRNGceiUPUpYMzPLur4EQYukQyPiCQBJhwFtfXifipT1un6WocBhwFygBnhE0qMR8cK73hSxAFgAUF9f3/sYg0JTazsA3zh5BmfNGZxhZmaVqS9B8BXgZklrk+19gDP78L5GYHLBdi2wtsg+b0REC7nA+SNwMPACFaa5JRcEXjbCzMrNdoMgIh6TNAP4K3Lf8p+LiI4+HPsxYLqkacArwFnkxgQK/Qa4QtJQYBi5U0f/sgP1HzSakx6BB4bNrNxsd/qopPOBURGxNCKeAUZL+m/be19EbAEuAO4ClgE3RUSDpPmS5if7LAPuBJ4GFgHXRsTSnW9O+WpKegTjRvmaATMrL305NfT5iLgyvxERzZI+T27a5zZFxB3AHb3Kru61/UPgh32r7uDVfWrIPQIzKzN9uaBsiKTugd/k+gD/NdtBza0dSLB7jXsEZlZe+tIjuAu4SdLV5Gb9zAf+kGqtKsy1f1rFrx5vZPeaaoZWeVUPMysvfQmCC8nN4f8CucHiJ8nNHLI++veHVrOpo5MzD5+8/Z3NzAbYdr+eJktLPAqsAurJzflflnK9Kkpzazsff/8kvnHyzFJXxczsPbbaI5B0ALkpn2cDG4BfAkTEBwemapVhU0cnre2dXkbCzMrWtk4NPQf8CfhoRKwAkPTVAalVBdnYmrvkwrOFzKxcbevU0N8B64D7JV0jaS7Fl42wbWjqnjbq2UJmVp62GgQRcWtEnAnMAB4AvgpMlPSvkk4YoPoNet1XFPvUkJmVqb4MFrdExM8i4hRy6wUtAYreZMbeq+eKYgeBmZWnvkwf7RYRTcC/JY9BbeGiNdz4yEupf87GpEfg21GaWbnaoSCoJHcsXccrG9s4vG5cqp+z79gaTpi9N3uOHp7q55iZ7azMBkFzSzvvnzKWa+fVl7oqZmYlldn1Dppb2xnnKZ1mZhkOgpZ2z+QxMyOjQbCpo5MWX+1rZgZkNAjyV/t6Jo+ZWUaDoHtuv8cIzMyyGQQ9c/sdBGZmmQyCzVu6AKgZVlXimpiZlV4mg6ArAoAhXkLPzCyrQZD7OUROAjOzjAZBLgmcA2ZmGQ2C6D415CQwM8tkEPjUkJlZj4wGgQeLzczyMhoEuZ9yj8DMLJtBEO4RmJl1y2QQ9MwachKYmWUzCHIXFrtHYGZGVoPA00fNzLplMgiie7C4tPUwMysHmQwC9wjMzHpkNAhyPx0EZmaZDQJPHzUzy8tkEISnj5qZdctmECQ/3SMwM8toEHR1ebDYzCwvm0HgwWIzs26pBoGkkyQ9L2mFpIu2sd/hkjolnZ5mffK6l5jIZAyamb1ban8KJVUBVwInA7OAsyXN2sp+PwDuSqsuvYV7BGZm3dL8TjwHWBERqyKiHVgInFpkvy8CtwCvp1iXd/H0UTOzHmkGwSTg5YLtxqSsm6RJwGnA1SnW4z08RmBm1iPNICj2VzZ6bf8EuDAiOrd5IOlcSYslLV6/fv0uV8w3rzcz6zE0xWM3ApMLtmuBtb32qQcWJhd2TQA+ImlLRNxWuFNELAAWANTX1/cOkx3mm9ebmfVIMwgeA6ZLmga8ApwFfLJwh4iYln8u6Xrg9t4hkAafGjIz65FaEETEFkkXkJsNVAVcFxENkuYnrw/ouEAhDxabmfVIs0dARNwB3NGrrGgARMQ5adalkG9eb2bWI5OXVEWEewNmZolMBkFXhMcHzMwSGQ0CDxSbmeVlNAjC1xCYmSUyGQThHoGZWbdMBkFXlweLzczyshkE7hGYmXXLaBB4jMDMLC+TQRARDPG5ITMzIKNB4FNDZmY9MhoEHiw2M8vLaBB4nSEzs7xMBoHXGjIz65HJIPBaQ2ZmPTIaBB4sNjPLy2gQ+DoCM7O8TAaB1xoyM+uRySBwj8DMrEdGg8A9AjOzvIwGgXsEZmZ5mQyC8PRRM7NumQyCri58QZmZWSKbQeAegZlZt4wGgdcaMjPLy2QQeK0hM7MemQwCnxoyM+uR0SDwYLGZWV5GgyA8RmBmlshkEIR7BGZm3TIZBB4jMDPr4SAwM8u4jAYBXmvIzCyRySDwWkNmZj2GlroCpdAVMCSTEWhWnjo6OmhsbGTTpk2lrsqgN2LECGpra6muru7zezIZBO4RmJWXxsZGxowZQ11dnad274KIYMOGDTQ2NjJt2rQ+vy+T34u91pBZedm0aRPjx4/3v8tdJInx48fvcM8qk0HgtYbMyo9DoH/szO8xk0HgW1WamfXIaBC4R2BmlpdqEEg6SdLzklZIuqjI65+S9HTyeFjSwWnWJ89jBGZmPVILAklVwJXAycAs4GxJs3rt9iLwNxFxEPAdYEFa9SnkMQIz25YLLriAqVOnlroaAybNHsEcYEVErIqIdmAhcGrhDhHxcEQ0J5uPArUp1qebl5gws6158cUXeeCBB2hvb+ftt99O7XM6OztTO/aOSvM6gknAywXbjcAR29j/s8Afir0g6VzgXIApU6bscsU8WGxWvi79XQPPrn2rX485a9/duOSjs/u07yWXXMLFF1/MNddcQ0NDA0ceeSQAa9eu5Ytf/CKrVq2ira2NG2+8kdra2veUzZkzhyOPPJKFCxdSV1fHK6+8wqmnnsrixYs544wzmDx5Mk8++SRz585lxowZ/OhHP6KtrY0xY8Zw6623sueeexb9rJqaGubPn89DDz0EwBNPPMHXv/517rvvvl3+/aQZBMX+0kbRHaUPkguCo4u9HhELSE4b1dfXFz3Gjsjdj2BXj2JmlaahoYGlS5dyww038Oc//7k7CLZs2cLJJ5/Md7/7XU455RRaW1vp7Ozk6KOPfk9ZRLBmzZruU0tPP/00Bx54IADPPPMMM2fO5P777wdgw4YNnH766QBceuml3HTTTZx33nlFP2vUqFGsXLmSzs5Oqqqq+NrXvsZll13WL+1OMwgagckF27XA2t47SToIuBY4OSI2pFifbuEegVnZ6us39zR861vf4jvf+Q6SmDlzJkuXLgXgtttuY+bMmZxyyikAjBw5kl/96lfvKQNYvnw506ZN656Qkg+CTZs20dTUxLe//e3uz7v++uv55S9/yebNm1m3bh3f+973in5W3uzZs2loaGD58uVMmTKFQw89tF/anWYQPAZMlzQNeAU4C/hk4Q6SpgC/Bv5LRLyQYl3exdNHzay3v/zlL9x1110sWbKE888/n02bNnHQQQcBsGTJku5TRHnFyiD3rT/fAwBYvHgx5513Hg0NDRxxxBEMHZr7s3vjjTeyaNEi7rvvPkaPHs2xxx7L7Nmzuf3224seF+DII4/koYce4qqrruLOO+/sr6anN1gcEVuAC4C7gGXATRHRIGm+pPnJbt8GxgNXSVoiaXFa9SnkwWIz6+2b3/wmt99+O6tXr2b16tU89dRT3T2Cvffem4aGhu59169fX7QMoKmpiZqaGgCWLVvG73//ew488ECeeeaZ7mCBXGAcddRRjB49mltuuYWHH36YAw88cKvHhVwQXHzxxZx22mlMmjSp39qe6nUEEXFHRBwQEftFxHeTsqsj4urk+eciYo+IOCR51KdZn7yuLl9HYGY97rnnHjZv3szcuXO7yyZOnEhLSwtNTU2cc845vPbaa8yePZtDDjmERx55pGgZwIknnsi9997LJz7xCW6++WbGjx/PxIkT3xME8+bN4/LLL+eYY47hhRde4H3vex+jRo3a6nEBZsyYwfDhw7nwwgv7tf2K2OWx1wFVX18fixfvWsfhqH+6lw/sP4EfnjEg16+Z2XYsW7aMmTNnlroaZe+CCy7g8MMPZ968edvcr9jvU9LjW/uynZklJh58YT3H//hBjv/xg7z29mbPGjKzQWPlypXMmDGDtra27YbAzsjM/QhGDx/K9ImjAThg4hhOe/+AXLtmZrbL9ttvP5577rnUjp+ZIDhs6h4cNvWwUlfDzKzsZObUkJmZFecgMLOyMNgmrpSrnfk9OgjMrORGjBjBhg0bHAa7KH/P4hEjRuzQ+zIzRmBm5au2tpbGxsZ3XTxlO2fEiBHU1u7YZBgHgZmVXHV1NdOmTSt1NTLLp4bMzDLOQWBmlnEOAjOzjBt0aw1JWg+8tJNvnwC80Y/VGQzc5mxwm7NhV9o8NSL2LPbCoAuCXSFp8UCtcFou3OZscJuzIa02+9SQmVnGOQjMzDIua0GwoNQVKAG3ORvc5mxIpc2ZGiMwM7P3ylqPwMzMenEQmJllXGaCQNJJkp6XtELSRaWuT3+RdJ2k1yUtLSgbJ+keScuTn3sUvPaN5HfwvKQTS1PrXSNpsqT7JS2T1CDpy0l5xbZb0ghJiyQ9lbT50qS8YtsMIKlK0pOSbk+2K7q9AJJWS3pG0hJJi5OydNsdERX/AKqAlcD7gGHAU8CsUtern9p2LHAosLSg7J+Bi5LnFwE/SJ7PSto+HJiW/E6qSt2GnWjzPsChyfMxwAtJ2yq23YCA0cnzauAvwJGV3OakHX8P/By4Pdmu6PYmbVkNTOhVlmq7s9IjmAOsiIhVEdEOLAROLXGd+kVE/BFo6lV8KnBD8vwG4OMF5QsjYnNEvAisIPe7GVQi4tWIeCJ5/jawDJhEBbc7ct5JNquTR1DBbZZUC/wtcG1BccW2dztSbXdWgmAS8HLBdmNSVqkmRsSrkPujCeyVlFfc70FSHfB+ct+QK7rdyWmSJcDrwD0RUelt/gnw34GugrJKbm9eAHdLelzSuUlZqu3Oyv0IVKQsi/NmK+r3IGk0cAvwlYh4SyrWvNyuRcoGXbsjohM4RNJY4FZJf72N3Qd1myWdArweEY9LOq4vbylSNmja28sHImKtpL2AeyQ9t419+6XdWekRNAKTC7ZrgbUlqstAeE3SPgDJz9eT8or5PUiqJhcCP4uIXyfFFd9ugIjYCDwAnETltvkDwMckrSZ3KvdDkv4fldvebhGxNvn5OnAruVM9qbY7K0HwGDBd0jRJw4CzgN+WuE5p+i0wL3k+D/hNQflZkoZLmgZMBxaVoH67RLmv/v8HWBYRPy54qWLbLWnPpCeApBrgw8BzVGibI+IbEVEbEXXk/r3eFxH/mQptb56kUZLG5J8DJwBLSbvdpR4hH8CR+I+Qm12yEvhWqevTj+36BfAq0EHu28FngfHAvcDy5Oe4gv2/lfwOngdOLnX9d7LNR5Pr/j4NLEkeH6nkdgMHAU8mbV4KfDspr9g2F7TjOHpmDVV0e8nNbHwqeTTk/1al3W4vMWFmlnFZOTVkZmZb4SAwM8s4B4GZWcY5CMzMMs5BYGaWcQ4Cs14kdSYrP+Yf/bZaraS6wpVizcpBVpaYMNsRbRFxSKkrYTZQ3CMw66NknfgfJPcFWCRp/6R8qqR7JT2d/JySlE+UdGtyD4GnJB2VHKpK0jXJfQXuTq4UNisZB4HZe9X0OjV0ZsFrb0XEHOAKcqtjkjy/MSIOAn4GXJ6UXw48GBEHk7tnRENSPh24MiJmAxuBv0u5PWbb5CuLzXqR9E5EjC5Svhr4UESsSha9WxcR4yW9AewTER1J+asRMUHSeqA2IjYXHKOO3BLS05PtC4HqiPhf6bfMrDj3CMx2TGzl+db2KWZzwfNOPFZnJeYgMNsxZxb8fCR5/jC5FTIBPgX8OXl+L/AF6L6pzG4DVUmzHeFvImbvVZPcCSzvzojITyEdLukv5L5EnZ2UfQm4TtI/AOuBzyTlXwYWSPosuW/+XyC3UqxZWfEYgVkfJWME9RHxRqnrYtaffGrIzCzj3CMwM8s49wjMzDLOQWBmlnEOAjOzjHMQmJllnIPAzCzj/gPsKFltpymnPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 利用鸢尾花数据集，实现前向传播、反向传播，可视化loss曲线\n",
    "\n",
    "# 导入所需模块\n",
    "import tensorflow as tf\n",
    "from sklearn import datasets\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import time  ##1##\n",
    "\n",
    "# 导入数据，分别为输入特征和标签\n",
    "x_data = datasets.load_iris().data\n",
    "y_data = datasets.load_iris().target\n",
    "\n",
    "# 随机打乱数据（因为原始数据是顺序的，顺序不打乱会影响准确率）\n",
    "# seed: 随机数种子，是一个整数，当设置之后，每次生成的随机数都一样（为方便教学，以保每位同学结果一致）\n",
    "np.random.seed(116)  # 使用相同的seed，保证输入特征和标签一一对应\n",
    "np.random.shuffle(x_data)\n",
    "np.random.seed(116)\n",
    "np.random.shuffle(y_data)\n",
    "tf.random.set_seed(116)\n",
    "\n",
    "# 将打乱后的数据集分割为训练集和测试集，训练集为前120行，测试集为后30行\n",
    "x_train = x_data[:-30]\n",
    "y_train = y_data[:-30]\n",
    "x_test = x_data[-30:]\n",
    "y_test = y_data[-30:]\n",
    "\n",
    "# 转换x的数据类型，否则后面矩阵相乘时会因数据类型不一致报错\n",
    "x_train = tf.cast(x_train, tf.float32)\n",
    "x_test = tf.cast(x_test, tf.float32)\n",
    "\n",
    "# from_tensor_slices函数使输入特征和标签值一一对应。（把数据集分批次，每个批次batch组数据）\n",
    "train_db = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(32)\n",
    "test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)\n",
    "\n",
    "# 生成神经网络的参数，4个输入特征故，输入层为4个输入节点；因为3分类，故输出层为3个神经元\n",
    "# 用tf.Variable()标记参数可训练\n",
    "# 使用seed使每次生成的随机数相同（方便教学，使大家结果都一致，在现实使用时不写seed）\n",
    "w1 = tf.Variable(tf.random.truncated_normal([4, 3], stddev=0.1, seed=1))\n",
    "b1 = tf.Variable(tf.random.truncated_normal([3], stddev=0.1, seed=1))\n",
    "\n",
    "lr = 0.1  # 学习率为0.1\n",
    "train_loss_results = []  # 将每轮的loss记录在此列表中，为后续画loss曲线提供数据\n",
    "test_acc = []  # 将每轮的acc记录在此列表中，为后续画acc曲线提供数据\n",
    "epoch = 500  # 循环500轮\n",
    "loss_all = 0  # 每轮分4个step，loss_all记录四个step生成的4个loss的和\n",
    "\n",
    "# 训练部分\n",
    "now_time = time.time()  ##2##\n",
    "for epoch in range(epoch):  # 数据集级别的循环，每个epoch循环一次数据集\n",
    "    for step, (x_train, y_train) in enumerate(train_db):  # batch级别的循环 ，每个step循环一个batch\n",
    "        with tf.GradientTape() as tape:  # with结构记录梯度信息\n",
    "            y = tf.matmul(x_train, w1) + b1  # 神经网络乘加运算\n",
    "            y = tf.nn.softmax(y)  # 使输出y符合概率分布（此操作后与独热码同量级，可相减求loss）\n",
    "            y_ = tf.one_hot(y_train, depth=3)  # 将标签值转换为独热码格式，方便计算loss和accuracy\n",
    "            loss = tf.reduce_mean(tf.square(y_ - y))  # 采用均方误差损失函数mse = mean(sum(y-out)^2)\n",
    "            loss_all += loss.numpy()  # 将每个step计算出的loss累加，为后续求loss平均值提供数据，这样计算的loss更准确\n",
    "        # 计算loss对各个参数的梯度\n",
    "        grads = tape.gradient(loss, [w1, b1])\n",
    "\n",
    "        # 实现梯度更新 w1 = w1 - lr * w1_grad    b = b - lr * b_grad\n",
    "        w1.assign_sub(lr * grads[0])  # 参数w1自更新\n",
    "        b1.assign_sub(lr * grads[1])  # 参数b自更新\n",
    "\n",
    "    # 每个epoch，打印loss信息\n",
    "    print(\"Epoch {}, loss: {}\".format(epoch, loss_all / 4))\n",
    "    train_loss_results.append(loss_all / 4)  # 将4个step的loss求平均记录在此变量中\n",
    "    loss_all = 0  # loss_all归零，为记录下一个epoch的loss做准备\n",
    "\n",
    "    # 测试部分\n",
    "    # total_correct为预测对的样本个数, total_number为测试的总样本数，将这两个变量都初始化为0\n",
    "    total_correct, total_number = 0, 0\n",
    "    for x_test, y_test in test_db:\n",
    "        # 使用更新后的参数进行预测\n",
    "        y = tf.matmul(x_test, w1) + b1\n",
    "        y = tf.nn.softmax(y)\n",
    "        pred = tf.argmax(y, axis=1)  # 返回y中最大值的索引，即预测的分类\n",
    "        # 将pred转换为y_test的数据类型\n",
    "        pred = tf.cast(pred, dtype=y_test.dtype)\n",
    "        # 若分类正确，则correct=1，否则为0，将bool型的结果转换为int型\n",
    "        correct = tf.cast(tf.equal(pred, y_test), dtype=tf.int32)\n",
    "        # 将每个batch的correct数加起来\n",
    "        correct = tf.reduce_sum(correct)\n",
    "        # 将所有batch中的correct数加起来\n",
    "        total_correct += int(correct)\n",
    "        # total_number为测试的总样本数，也就是x_test的行数，shape[0]返回变量的行数\n",
    "        total_number += x_test.shape[0]\n",
    "    # 总的准确率等于total_correct/total_number\n",
    "    acc = total_correct / total_number\n",
    "    test_acc.append(acc)\n",
    "    print(\"Test_acc:\", acc)\n",
    "    print(\"--------------------------\")\n",
    "total_time = time.time() - now_time  ##3##\n",
    "print(\"total_time\", total_time)  ##4##\n",
    "\n",
    "# 绘制 loss 曲线\n",
    "plt.title('Loss Function Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Loss')  # y轴变量名称\n",
    "plt.plot(train_loss_results, label=\"$Loss$\")  # 逐点画出trian_loss_results值并连线，连线图标是Loss\n",
    "plt.legend()  # 画出曲线图标\n",
    "plt.show()  # 画出图像\n",
    "\n",
    "# 绘制 Accuracy 曲线\n",
    "plt.title('Acc Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Acc')  # y轴变量名称\n",
    "plt.plot(test_acc, label=\"$Accuracy$\")  # 逐点画出test_acc值并连线，连线图标是Accuracy\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# 本文件较 class1\\p45_iris.py 仅添加四处时间记录  用 ##n## 标识\n",
    "# 请将loss曲线、ACC曲线、total_time记录到 class2\\优化器对比.docx  对比各优化器收敛情况\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SGDM算法-相较于SGD增加了一阶动量（含momentum的SGD）\n",
    "$$ m_t = \\beta *m_{t-1}+(1-\\beta)g_t $$\n",
    "$$ V_t = 1 $$\n",
    "$$ \\eta _t =\\frac{lr*m_t}{\\sqrt{V_t}} = lr*m_t = lr*{\\beta *m_{t-1}+(1-\\beta)g_t} $$\n",
    "$$ w_{t+1} = w_t - lr*{\\beta *m_{t-1}+(1-\\beta)g_t} $$\n",
    "\n",
    "        beta = 0.9  \n",
    "        m_w = beta * m_w + (1 - beta) * grads[0]  \n",
    "        m_b = beta * m_b + (1 - beta) * grads[1]  \n",
    "        w1.assign_sub(lr * m_w)  \n",
    "        b1.assign_sub(lr * m_b)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, loss: 0.2961867228150368\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 1, loss: 0.28081152960658073\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 2, loss: 0.26392311602830887\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 3, loss: 0.241925410926342\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 4, loss: 0.2185598611831665\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 5, loss: 0.20465287193655968\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 6, loss: 0.1955692023038864\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 7, loss: 0.18339980021119118\n",
      "Test_acc: 0.5\n",
      "--------------------------\n",
      "Epoch 8, loss: 0.17289477586746216\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 9, loss: 0.1638195738196373\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 10, loss: 0.15572058036923409\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 11, loss: 0.14825156331062317\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 12, loss: 0.14126422069966793\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 13, loss: 0.1357906088232994\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 14, loss: 0.13151294365525246\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 15, loss: 0.12771007791161537\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 16, loss: 0.12420250289142132\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 17, loss: 0.12114136666059494\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 18, loss: 0.11857617646455765\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 19, loss: 0.11636174656450748\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 20, loss: 0.11436180211603642\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 21, loss: 0.11252963356673717\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 22, loss: 0.11085251905024052\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 23, loss: 0.10932632349431515\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 24, loss: 0.10794146731495857\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 25, loss: 0.10666536539793015\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 26, loss: 0.10546092130243778\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 27, loss: 0.10430946573615074\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 28, loss: 0.10321032628417015\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 29, loss: 0.10216889530420303\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 30, loss: 0.10118389315903187\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 31, loss: 0.10024458728730679\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 32, loss: 0.09933912195265293\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 33, loss: 0.09846181236207485\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 34, loss: 0.09761266224086285\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 35, loss: 0.09679244831204414\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 36, loss: 0.09599893540143967\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 37, loss: 0.09522782824933529\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 38, loss: 0.09447569027543068\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 39, loss: 0.09374143369495869\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 40, loss: 0.0930252056568861\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 41, loss: 0.09232662431895733\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 42, loss: 0.09164421819150448\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 43, loss: 0.09097621589899063\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 44, loss: 0.090321509167552\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 45, loss: 0.08967970311641693\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 46, loss: 0.0890505388379097\n",
      "Test_acc: 0.6666666666666666\n",
      "--------------------------\n",
      "Epoch 47, loss: 0.08843343332409859\n",
      "Test_acc: 0.6666666666666666\n",
      "--------------------------\n",
      "Epoch 48, loss: 0.08782758750021458\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 49, loss: 0.08723233081400394\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 50, loss: 0.08664725534617901\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 51, loss: 0.08607210591435432\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 52, loss: 0.08550658822059631\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 53, loss: 0.08495028875768185\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 54, loss: 0.08440282009541988\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 55, loss: 0.08386384323239326\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 56, loss: 0.08333314396440983\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 57, loss: 0.08281049504876137\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 58, loss: 0.08229565247893333\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 59, loss: 0.08178837038576603\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 60, loss: 0.08128841780126095\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 61, loss: 0.08079558797180653\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 62, loss: 0.08030974678695202\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 63, loss: 0.07983068935573101\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 64, loss: 0.07935825921595097\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 65, loss: 0.07889229245483875\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 66, loss: 0.07843263447284698\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 67, loss: 0.07797915115952492\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 68, loss: 0.07753170095384121\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 69, loss: 0.07709016837179661\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 70, loss: 0.07665440626442432\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 71, loss: 0.07622430566698313\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 72, loss: 0.07579975295811892\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 73, loss: 0.07538064103573561\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 74, loss: 0.07496685814112425\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 75, loss: 0.07455830369144678\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 76, loss: 0.07415488641709089\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 77, loss: 0.07375650573521852\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 78, loss: 0.07336306478828192\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 79, loss: 0.0729744816198945\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 80, loss: 0.07259066589176655\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 81, loss: 0.07221154309809208\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 82, loss: 0.07183701451867819\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 83, loss: 0.07146701496094465\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 84, loss: 0.07110146433115005\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 85, loss: 0.07074028719216585\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 86, loss: 0.07038341462612152\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 87, loss: 0.0700307684019208\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 88, loss: 0.0696822851896286\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 89, loss: 0.06933789234608412\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 90, loss: 0.06899752654135227\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 91, loss: 0.06866111885756254\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 92, loss: 0.06832861807197332\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 93, loss: 0.06799995433539152\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 94, loss: 0.06767506990581751\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 95, loss: 0.06735390704125166\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 96, loss: 0.0670364061370492\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 97, loss: 0.06672250758856535\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 98, loss: 0.06641216110438108\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 99, loss: 0.0661053080111742\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 100, loss: 0.06580189801752567\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 101, loss: 0.06550187896937132\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 102, loss: 0.06520519964396954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 103, loss: 0.06491181906312704\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 104, loss: 0.06462167203426361\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 105, loss: 0.06433472037315369\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 106, loss: 0.06405091844499111\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 107, loss: 0.06377020850777626\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 108, loss: 0.06349255610257387\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 109, loss: 0.06321791838854551\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 110, loss: 0.06294624414294958\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 111, loss: 0.06267749331891537\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 112, loss: 0.06241162307560444\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 113, loss: 0.062148584984242916\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 114, loss: 0.061888355761766434\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 115, loss: 0.06163087673485279\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 116, loss: 0.0613761255517602\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 117, loss: 0.06112405098974705\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 118, loss: 0.06087461858987808\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 119, loss: 0.06062779109925032\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 120, loss: 0.06038354150950909\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 121, loss: 0.06014181114733219\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 122, loss: 0.059902590699493885\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 123, loss: 0.0596658093854785\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 124, loss: 0.05943147744983435\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 125, loss: 0.059199538081884384\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 126, loss: 0.05896995775401592\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 127, loss: 0.058742702938616276\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 128, loss: 0.05851774662733078\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 129, loss: 0.0582950534299016\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 130, loss: 0.05807459447532892\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 131, loss: 0.05785634275525808\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 132, loss: 0.05764026381075382\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 133, loss: 0.05742631945759058\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 134, loss: 0.05721450038254261\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 135, loss: 0.05700476095080376\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 136, loss: 0.05679708439856768\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 137, loss: 0.05659143999218941\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 138, loss: 0.05638779606670141\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 139, loss: 0.0561861302703619\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 140, loss: 0.055986412800848484\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 141, loss: 0.055788625963032246\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 142, loss: 0.0555927325040102\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 143, loss: 0.05539871007204056\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 144, loss: 0.05520654562860727\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 145, loss: 0.05501619540154934\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 146, loss: 0.054827659390866756\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 147, loss: 0.05464089661836624\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 148, loss: 0.05445587821304798\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 149, loss: 0.05427259858697653\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 150, loss: 0.05409103445708752\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 151, loss: 0.05391115974634886\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 152, loss: 0.053732942789793015\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 153, loss: 0.05355636775493622\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 154, loss: 0.0533814188092947\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 155, loss: 0.053208074532449245\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 156, loss: 0.05303630419075489\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 157, loss: 0.05286610499024391\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 158, loss: 0.052697448059916496\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 159, loss: 0.05253030639141798\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 160, loss: 0.05236467346549034\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 161, loss: 0.052200524136424065\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 162, loss: 0.05203784257173538\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 163, loss: 0.05187660921365023\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 164, loss: 0.05171679798513651\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 165, loss: 0.051558395847678185\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 166, loss: 0.051401397213339806\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 167, loss: 0.05124577786773443\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 168, loss: 0.051091515459120274\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 169, loss: 0.050938598811626434\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 170, loss: 0.05078699626028538\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 171, loss: 0.050636718049645424\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 172, loss: 0.050487727858126163\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 173, loss: 0.05034002009779215\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 174, loss: 0.050193569622933865\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 175, loss: 0.05004837550222874\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 176, loss: 0.04990440979599953\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 177, loss: 0.04976166784763336\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 178, loss: 0.04962012451142073\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 179, loss: 0.0494797695428133\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 180, loss: 0.04934058617800474\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 181, loss: 0.049202573485672474\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 182, loss: 0.04906571004539728\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 183, loss: 0.04892996698617935\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 184, loss: 0.04879535548388958\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 185, loss: 0.0486618485301733\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 186, loss: 0.04852943401783705\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 187, loss: 0.048398105427622795\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 188, loss: 0.0482678422704339\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 189, loss: 0.04813864175230265\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 190, loss: 0.04801047686487436\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 191, loss: 0.04788334295153618\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 192, loss: 0.04775724932551384\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 193, loss: 0.04763214290142059\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 194, loss: 0.04750804044306278\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 195, loss: 0.047384921461343765\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 196, loss: 0.04726278316229582\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 197, loss: 0.04714160691946745\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 198, loss: 0.047021384350955486\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 199, loss: 0.046902100555598736\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 200, loss: 0.04678374528884888\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 201, loss: 0.046666317619383335\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 202, loss: 0.04654979985207319\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 203, loss: 0.04643417801707983\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 204, loss: 0.046319447457790375\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 205, loss: 0.04620560072362423\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 206, loss: 0.0460926229134202\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 207, loss: 0.045980511233210564\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 208, loss: 0.04586923308670521\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 209, loss: 0.04575882013887167\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 210, loss: 0.04564923234283924\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 211, loss: 0.0455404669046402\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 212, loss: 0.04543251823633909\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 213, loss: 0.04532537516206503\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 214, loss: 0.04521902557462454\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 215, loss: 0.04511347133666277\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 216, loss: 0.045008692890405655\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 217, loss: 0.044904688373208046\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 218, loss: 0.044801450334489346\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 219, loss: 0.04469895828515291\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 220, loss: 0.044597224332392216\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 221, loss: 0.044496238231658936\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 222, loss: 0.04439597111195326\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 223, loss: 0.04429642576724291\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 224, loss: 0.04419760126620531\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 225, loss: 0.04409949108958244\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 226, loss: 0.04400208033621311\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 227, loss: 0.043905358761548996\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 228, loss: 0.043809326365590096\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 229, loss: 0.043713973835110664\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 230, loss: 0.04361929465085268\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 231, loss: 0.043525281362235546\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 232, loss: 0.043431926518678665\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 233, loss: 0.04333922918885946\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 234, loss: 0.04324717819690704\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 235, loss: 0.04315576236695051\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 236, loss: 0.043064977042376995\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 237, loss: 0.04297481942921877\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 238, loss: 0.04288528487086296\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 239, loss: 0.042796362191438675\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 240, loss: 0.04270805045962334\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 241, loss: 0.04262033849954605\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 242, loss: 0.0425332160666585\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 243, loss: 0.0424466896802187\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 244, loss: 0.042360746301710606\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 245, loss: 0.04227538127452135\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 246, loss: 0.042190590873360634\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 247, loss: 0.042106364853680134\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 248, loss: 0.04202269110828638\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 249, loss: 0.041939590126276016\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 250, loss: 0.04185703629627824\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 251, loss: 0.04177501704543829\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 252, loss: 0.04169354401528835\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 253, loss: 0.04161259951069951\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 254, loss: 0.04153219098225236\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 255, loss: 0.04145230073481798\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 256, loss: 0.04137293063104153\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 257, loss: 0.04129408160224557\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 258, loss: 0.04121573781594634\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 259, loss: 0.04113789135590196\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 260, loss: 0.04106055432930589\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 261, loss: 0.04098370671272278\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 262, loss: 0.04090734990313649\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 263, loss: 0.04083147598430514\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 264, loss: 0.040756091475486755\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 265, loss: 0.04068117821589112\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 266, loss: 0.04060674179345369\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 267, loss: 0.040532763581722975\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 268, loss: 0.040459245909005404\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 269, loss: 0.040386196691542864\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 270, loss: 0.04031360521912575\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 271, loss: 0.040241456124931574\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 272, loss: 0.04016975173726678\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 273, loss: 0.0400984943844378\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 274, loss: 0.04002768034115434\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 275, loss: 0.039957290049642324\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 276, loss: 0.039887330029159784\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 277, loss: 0.03981780353933573\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 278, loss: 0.03974868869408965\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 279, loss: 0.039680005982518196\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 280, loss: 0.03961173119023442\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 281, loss: 0.039543870370835066\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 282, loss: 0.039476416539400816\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 283, loss: 0.0394093724898994\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 284, loss: 0.039342711213976145\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 285, loss: 0.0392764606513083\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 286, loss: 0.0392106007784605\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 287, loss: 0.03914513532072306\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 288, loss: 0.03908004518598318\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 289, loss: 0.03901534387841821\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 290, loss: 0.03895101975649595\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 291, loss: 0.03888707049190998\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 292, loss: 0.03882350539788604\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 293, loss: 0.03876029932871461\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 294, loss: 0.03869746392592788\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 295, loss: 0.03863498894497752\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 296, loss: 0.038572877645492554\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 297, loss: 0.03851112071424723\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 298, loss: 0.03844971628859639\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 299, loss: 0.038388669956475496\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 300, loss: 0.03832796588540077\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 301, loss: 0.038267606403678656\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 302, loss: 0.038207584992051125\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 303, loss: 0.03814791375771165\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 304, loss: 0.03808857314288616\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 305, loss: 0.03802956873551011\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 306, loss: 0.037970895413309336\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 307, loss: 0.0379125471226871\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 308, loss: 0.037854521069675684\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 309, loss: 0.03779682004824281\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 310, loss: 0.03773943893611431\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 311, loss: 0.03768237493932247\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 312, loss: 0.03762562479823828\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 313, loss: 0.037569188978523016\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 314, loss: 0.037513059098273516\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 315, loss: 0.03745723515748978\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 316, loss: 0.03740171901881695\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 317, loss: 0.03734650555998087\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 318, loss: 0.03729158779606223\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 319, loss: 0.03723696619272232\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 320, loss: 0.03718263702467084\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 321, loss: 0.03712860634550452\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 322, loss: 0.037074866238981485\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 323, loss: 0.03702140832319856\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 324, loss: 0.036968246567994356\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 325, loss: 0.03691535023972392\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 326, loss: 0.03686273889616132\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 327, loss: 0.036810411140322685\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 328, loss: 0.0367583567276597\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 329, loss: 0.03670658590272069\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 330, loss: 0.03665507631376386\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 331, loss: 0.03660383401438594\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 332, loss: 0.03655285993590951\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 333, loss: 0.03650216246023774\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 334, loss: 0.03645171318203211\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 335, loss: 0.03640153491869569\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 336, loss: 0.036351611372083426\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 337, loss: 0.036301950458437204\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 338, loss: 0.03625253541395068\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 339, loss: 0.03620337648317218\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 340, loss: 0.03615447273477912\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 341, loss: 0.036105822306126356\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 342, loss: 0.03605741122737527\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 343, loss: 0.03600924555212259\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 344, loss: 0.03596133179962635\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 345, loss: 0.035913648549467325\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 346, loss: 0.03586621209979057\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 347, loss: 0.03581901080906391\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 348, loss: 0.03577204421162605\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 349, loss: 0.035725321620702744\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 350, loss: 0.03567882254719734\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 351, loss: 0.03563255537301302\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 352, loss: 0.03558651590719819\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 353, loss: 0.03554071020334959\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 354, loss: 0.035495126619935036\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 355, loss: 0.03544977027922869\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 356, loss: 0.035404634196311235\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 357, loss: 0.03535971092060208\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 358, loss: 0.03531502140685916\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 359, loss: 0.03527053352445364\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 360, loss: 0.03522627288475633\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 361, loss: 0.035182226449251175\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 362, loss: 0.035138390492647886\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 363, loss: 0.035094758961349726\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 364, loss: 0.03505134070292115\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 365, loss: 0.035008135717362165\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 366, loss: 0.03496513469144702\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 367, loss: 0.03492234228178859\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 368, loss: 0.03487975196912885\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 369, loss: 0.03483736328780651\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 370, loss: 0.03479517204686999\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 371, loss: 0.03475317498669028\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 372, loss: 0.03471138747408986\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 373, loss: 0.03466978808864951\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 374, loss: 0.03462838986888528\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 375, loss: 0.03458718443289399\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 376, loss: 0.03454616945236921\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 377, loss: 0.03450534725561738\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 378, loss: 0.03446470759809017\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 379, loss: 0.034424257930368185\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 380, loss: 0.03438399871811271\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 381, loss: 0.03434392949566245\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 382, loss: 0.03430404048413038\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 383, loss: 0.034264333080500364\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 384, loss: 0.03422480775043368\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 385, loss: 0.03418545890599489\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 386, loss: 0.03414629679173231\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 387, loss: 0.0341073009185493\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 388, loss: 0.0340684917755425\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 389, loss: 0.034029860980808735\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 390, loss: 0.033991402480751276\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 391, loss: 0.0339531097561121\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 392, loss: 0.03391498886048794\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 393, loss: 0.03387704957276583\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 394, loss: 0.03383927559480071\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 395, loss: 0.03380166180431843\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 396, loss: 0.03376422077417374\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 397, loss: 0.0337269427254796\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 398, loss: 0.03368983929976821\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 399, loss: 0.0336528979241848\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 400, loss: 0.03361610742285848\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 401, loss: 0.03357949201017618\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 402, loss: 0.0335430265404284\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 403, loss: 0.03350672638043761\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 404, loss: 0.03347058407962322\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 405, loss: 0.033434598706662655\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 406, loss: 0.03339876979589462\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 407, loss: 0.03336309362202883\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 408, loss: 0.03332757716998458\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 409, loss: 0.03329221531748772\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 410, loss: 0.03325699595734477\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 411, loss: 0.033221937250345945\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 412, loss: 0.03318702010437846\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 413, loss: 0.033152253832668066\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 414, loss: 0.03311764448881149\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 415, loss: 0.03308317298069596\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 416, loss: 0.03304885374382138\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 417, loss: 0.03301468072459102\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 418, loss: 0.03298064973205328\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 419, loss: 0.03294675797224045\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 420, loss: 0.03291300963610411\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 421, loss: 0.03287941124290228\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 422, loss: 0.03284594463184476\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 423, loss: 0.032812617253512144\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 424, loss: 0.03277944074943662\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 425, loss: 0.032746391370892525\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 426, loss: 0.03271348820999265\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 427, loss: 0.03268071124330163\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 428, loss: 0.03264807444065809\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 429, loss: 0.032615567091852427\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 430, loss: 0.03258320456370711\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 431, loss: 0.03255096962675452\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 432, loss: 0.03251886274665594\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 433, loss: 0.0324868937022984\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 434, loss: 0.0324550517834723\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 435, loss: 0.032423331402242184\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 436, loss: 0.03239175118505955\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 437, loss: 0.03236029902473092\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 438, loss: 0.032328967936336994\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 439, loss: 0.032297768630087376\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 440, loss: 0.032266685739159584\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 441, loss: 0.032235737424343824\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 442, loss: 0.03220490925014019\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 443, loss: 0.032174210995435715\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 444, loss: 0.03214363195002079\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 445, loss: 0.03211316652595997\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 446, loss: 0.03208282636478543\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 447, loss: 0.03205260634422302\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 448, loss: 0.03202251344919205\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 449, loss: 0.031992537435144186\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 450, loss: 0.03196267085149884\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 451, loss: 0.03193293046206236\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 452, loss: 0.031903300900012255\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 453, loss: 0.03187378961592913\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 454, loss: 0.0318443956784904\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 455, loss: 0.031815124209970236\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 456, loss: 0.03178595006465912\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 457, loss: 0.031756890937685966\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 458, loss: 0.031727951020002365\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 459, loss: 0.031699119601398706\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 460, loss: 0.03167040226981044\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 461, loss: 0.03164179250597954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 462, loss: 0.03161329356953502\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 463, loss: 0.031584908720105886\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 464, loss: 0.03155662724748254\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 465, loss: 0.03152844915166497\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 466, loss: 0.03150038653984666\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 467, loss: 0.031472431030124426\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 468, loss: 0.03144457656890154\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 469, loss: 0.03141682827845216\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 470, loss: 0.031389190815389156\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 471, loss: 0.0313616506755352\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 472, loss: 0.03133421717211604\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 473, loss: 0.03130688378587365\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 474, loss: 0.03127965424209833\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 475, loss: 0.03125252714380622\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 476, loss: 0.03122550342231989\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 477, loss: 0.031198577489703894\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 478, loss: 0.03117175679653883\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 479, loss: 0.031145032960921526\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 480, loss: 0.03111840272322297\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 481, loss: 0.031091873534023762\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 482, loss: 0.031065434217453003\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 483, loss: 0.031039108522236347\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 484, loss: 0.031012879218906164\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 485, loss: 0.030986736994236708\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 486, loss: 0.030960693024098873\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 487, loss: 0.0309347384609282\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 488, loss: 0.030908886343240738\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 489, loss: 0.030883120372891426\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 490, loss: 0.030857461038976908\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 491, loss: 0.030831883661448956\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 492, loss: 0.030806399416178465\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 493, loss: 0.030781006440520287\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 494, loss: 0.030755708925426006\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 495, loss: 0.030730504542589188\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 496, loss: 0.030705390498042107\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 497, loss: 0.03068035189062357\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 498, loss: 0.03065542411059141\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 499, loss: 0.03063056943938136\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "total_time 10.354207992553711\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxc5X3v8c9Ps2i0y5ZkY8s2NrYTYoNZ4rCUJZBCAmQxKbkJaUqaFK7rJtyUV0huyE2aNkvvq7RZaUjZA1mApr2BGOKyBBp2gkXiHRwb22AhY0m2Ze377/5xjuSxPLJHtsYjz3zfr9e8Zs5zztE8j18w33nOc+Z5zN0REREZqSDbFRARkYlJASEiIikpIEREJCUFhIiIpKSAEBGRlBQQIiKSkgJCJAvMrN3MTsh2PUQORgEhWWNm28zsoiy8791m1ht+SA89PpbB9/utmV2TXObupe6+JUPv9+dmVhe2a4eZ/ZeZnZuJ95LcpoCQfPXP4Yf00OPfs12h8WBmnwe+D/xfYCowC/gRsOQw/lZ0fGsnxxoFhEw4ZlZoZt83s4bw8X0zKwz3VZvZw2bWYma7zewZMysI933JzN40szYz22hmfzrG973bzL6VtH2BmdUnbW8zsy+Y2Roz22tm/25miaT9S8xslZm1mtlrZnaJmf0jcB7ww/Ab/Q/DY93M5oWvK8zsJ2bWZGavm9lXk9r0KTN71sy+bWZ7zGyrmV06Sv0rgG8An3X3X7p7h7v3uftD7v7FMbTxS2a2BugI6/KfI97nB2Z2U1Ld7wx7Km+a2bfMLDKWf3eZuPQNQSairwBnAacCDvwK+Crwd8D1QD1QEx57FuBm9nbgWuBd7t5gZrOBTHxQfRS4BOgGngM+BdxiZmcAPwE+AjwBTAPK3P0RMzsH+Jm73zHK3/xXoAI4AagCHgN2AHeG+88E7gGqgaXAnWZW6wfOk3M2kAAeOMI2fhx4P9AMTAH+j5mVu3tr+OH/UeDD4bH3ADuBeUAJ8DCwHbj1COsgE4B6EDIRfQL4hrs3unsT8HXgqnBfH8GH7/Hht+Nnwg/KAaAQWGBmMXff5u6vHeQ9vhD2QlrMrHkMdbvJ3RvcfTfwEEGIAVwN3OXuj7v7oLu/6e6vHuqPhR+4HwO+7O5t7r4N+E5SewFed/fb3X2A4AN5GsHlo5GqgGZ37x9De1K5yd23u3uXu78O/B64PNz3HqDT3V80s6nApcB1YW+lEfgecOURvr9MEAoImYimA68nbb8elgH8C7AZeMzMtpjZDQDuvhm4DvgHoNHM7jez6Yzu2+5eGT6qx1C3t5JedwKl4euZwMECaTTVQJwD21ub6j3dvTN8WcqBdgHV4zB2sH3E9r0EvQqAPw+3AY4HYsCOobAl6DlMOcL3lwlCASETUQPBh8+QWWEZ4bfs6939BOCDwOeHxhrc/V53Pzc814Ebx/i+HUBx0vZxYzh3OzB3lH0HmzK5maBXNLK9b47hvYe8QHDp6/KDHJNOG0fW9z+AC8xsBsGlpaGA2A70ANVJYVvu7gsPo+4yASkgJNtiZpZIekSB+4CvmlmNmVUDXwN+BmBmHzCzeWZmQCvBpaUBM3u7mb0nHMzuBrrCfWOxCrjMzCab2XEEPZJ03Ql82sz+1MwKzKzWzE4M9+0kGF84QHjZ6BfAP5pZmZkdD3x+qL1j4e57Cf6tbjazy82s2MxiZnapmf3z4bYxvMz3W+DHwFZ3fyUs30EwXvIdMysP2z3XzN491rrLxKSAkGxbQfBhPvT4B+BbQB2wBlhLcA186M6b+cBvgHaCb8w/cvffEow//BPBN/K3CAdXx1iXnwKrgW0EH3xp3/rq7i8Bnya4Br8XeIp9vYIfAB8J70K6KcXp/4vgm/0W4FmCb+h3jbHuQ/X4LkHAfBVoIviWfy3wYHjI4bbxXuAi9vUehnyS4BLZBmAP8J8EYySSA0wLBomISCrqQYiISEoKCBERSUkBISIiKSkgREQkpZyaaqO6utpnz56d7WqIiBwzXn755WZ3r0m1L6cCYvbs2dTV1WW7GiIixwwze320fbrEJCIiKSkgREQkpYwGRDgf/kYz2zw0qdqI/UvCufVXWbAC1rnpnisiIpmVsTGIcBrjm4GLCebvX2lmy919Q9JhTwDL3d3NbBHBnDQnpnmuiMgR6evro76+nu7u7mxXJeMSiQQzZswgFoulfU4mB6nPADYPrbtrZvcTLHs4/CHv7u1Jx5ewbxbJQ54rInKk6uvrKSsrY/bs2QTzP+Ymd2fXrl3U19czZ86ctM/L5CWmWvafV76e/ee4B8DMPmxmrwK/Bv5qLOeG5y8NL0/VNTU1jUvFRSQ/dHd3U1VVldPhAGBmVFVVjbmnlMmASPUvfsDMgO7+gLufSDCH/TfHcm54/m3uvtjdF9fUpLyVV0RkVLkeDkMOp52ZDIh6glW2hswgXPQlFXd/Gpgbzv8/pnOP1E1PbOKpP6r3ISKSLJMBsRKYb2ZzzCxOsE7t8uQDkhZ+wcxOJ5hXflc6546n257ewlMbFRAiIskyNkjt7v1mdi3wKBAhWNB9vZktC/ffAlwBfNLM+ggWi/lYuAB9ynMzVdeSwggdPUe6zruISG7J6FQb7r6CYMWw5LJbkl7fyCjrBqc6N1NKCqO09yogRCQ7br31VtasWcPNN9+c7arsR7+kBkoLo+pBiEjWrFmzhpNPPjnb1TiAAgIoiSsgRCR71q5de0BAvPrqq5x//vksXLiQiy66iObmZgDuuece3vnOd7Jo0SLOO++8UcvGQ07N5nq4SgqjvNnSle1qiEgWff2h9WxoaB3Xv7lgejl//8GFhzxu3bp1nHTSScPbPT09XHHFFfzsZz/jtNNO48Ybb+R73/seN9xwAzfeeCOrVq0iHo/T0tJCW1vbAWXjRT0IoFSD1CKSJdu3b6esrIyKiorhsgcffJBzzz2X0047DYAFCxbQ2NhIJBKhq6uL66+/nrq6OiorK1OWjRf1IAh6EAoIkfyWzjf9TEg1/rBhw4b9ytauXcuCBQsoLi5m3bp1PPTQQyxdupRrrrmGz3zmMynLxoMCgmCQul0BISJZkGr8oba2llWrVgGwZcsWfvrTn/Lss8+yadMm5s+fz5VXXsmGDRvo7u5OWTZeFBAEPYie/kH6BwaJRnTVTUSOnrVr1/LII49w3333ATBt2jSefPJJVqxYwcknn0xRURF33XUXVVVVXH/99bzwwguUlJSwcOFCbr/9dpYtW3ZA2XhRQBAEBEBHzwAVxQoIETl6fv7zn6csf/DBBw8ou/vuu9MqGy/6NCQYpAb0YzkRkSQKCJJ7EAoIEZEhCgj2BYQGqkXyTzD9W+47nHYqIAjuYgL1IETyTSKRYNeuXTkfEkMryiUSiTGdp0Fqgqk2QAEhkm9mzJhBfX09+bAa5dCa1GOhgGBfD6K9ZyDLNRGRoykWi41pjeZ8o0tMBOtBgHoQIiLJFBBokFpEJBUFBFAYLSBaYOpBiIgkUUAAZqYJ+0RERlBAhIIJ+zRILSIyRAERKtGaECIi+1FAhEoKo3RoLiYRkWEKiJDWpRYR2Z8CIpSIFdDdN5jtaoiITBgKiFAiFqG7X4PUIiJDFBChRCxCj3oQIiLDFBCh4BKTehAiIkMUEKFENEKXAkJEZJgCIpSIRejuG8j5eeFFRNKV0YAws0vMbKOZbTazG1Ls/4SZrQkfz5vZKUn7tpnZWjNbZWZ1mawnQFE8wqBD34ACQkQEMrgehJlFgJuBi4F6YKWZLXf3DUmHbQXe7e57zOxS4DbgzKT9F7p7c6bqmKwwGmRld/8A8ag6ViIimfwkPAPY7O5b3L0XuB9YknyAuz/v7nvCzReBsS13NI4SsWBNiO5ejUOIiEBmA6IW2J60XR+WjeZq4L+Sth14zMxeNrOlGajffoYDQre6iogAmV1y1FKUpbzAb2YXEgTEuUnF57h7g5lNAR43s1fd/ekU5y4FlgLMmjXrsCtbNBQQ+rGciAiQ2R5EPTAzaXsG0DDyIDNbBNwBLHH3XUPl7t4QPjcCDxBcsjqAu9/m7ovdfXFNTc1hVzYRC8cgdKuriAiQ2YBYCcw3szlmFgeuBJYnH2Bms4BfAle5+x+TykvMrGzoNfBeYF0G6zp8ialLYxAiIkAGLzG5e7+ZXQs8CkSAu9x9vZktC/ffAnwNqAJ+ZGYA/e6+GJgKPBCWRYF73f2RTNUVknoQ/RqDEBGBzI5B4O4rgBUjym5Jen0NcE2K87YAp4wsz6R9g9TqQYiIgH5JPUwBISKyPwVEaCggNKOriEhAARFKhL+e1oR9IiIBBUSoKK5LTCIiyRQQoURUv6QWEUmmgAgVFBjxSIF+SS0iElJAJCmMFeiHciIiIQVEkqJYhB71IEREAAXEfoJV5TQGISICCoj9JGIFuotJRCSkgEiSiEX0OwgRkZACIkkiGlEPQkQkpIBIkohrDEJEZIgCIkkiqjEIEZEhCogkwV1MCggREVBA7Ce4i0mXmEREQAGxn6JYRFNtiIiEFBBJErGIptoQEQkpIJIUx6P09A/SP6DLTCIiCogkZYlgie72nv4s10REJPsUEEmGAqKtWwEhIqKASFKWiAHQ2t2X5ZqIiGSfAiJJuXoQIiLDFBBJhnoQCggREQXEfvaNQegSk4iIAiKJBqlFRPZRQCTZd4lJPQgREQVEkni0gMJoAa3qQYiIKCBGqiiKsbdTPQgRkYwGhJldYmYbzWyzmd2QYv8nzGxN+HjezE5J99xMqSotZFdHz9F6OxGRCStjAWFmEeBm4FJgAfBxM1sw4rCtwLvdfRHwTeC2MZybEdWlcZrae4/GW4mITGiZ7EGcAWx29y3u3gvcDyxJPsDdn3f3PeHmi8CMdM/NlJrSQprb1IMQEclkQNQC25O268Oy0VwN/NdYzzWzpWZWZ2Z1TU1NR1DdQHVZIc3tPbj7Ef8tEZFjWSYDwlKUpfzUNbMLCQLiS2M9191vc/fF7r64pqbmsCqarKa0kJ7+Qdo0o6uI5LlMBkQ9MDNpewbQMPIgM1sE3AEscfddYzk3E6rL4gC6zCQieS+TAbESmG9mc8wsDlwJLE8+wMxmAb8ErnL3P47l3EypKikEYHeHBqpFJL9FM/WH3b3fzK4FHgUiwF3uvt7MloX7bwG+BlQBPzIzgP7wclHKczNV12SVxcGvqVv0WwgRyXMZCwgAd18BrBhRdkvS62uAa9I992iYVBxcYtrTqR6EiOQ3/ZJ6hIqwB7G3Sz0IEclvCogRygqjRApMPQgRyXsKiBHMjMqimMYgRCTvKSBSqCxWQIiIKCBSqCyO09KlS0wikt8UEClMKo6xp0M9CBHJbwqIFCqK4rqLSUTyngIihUnFMd3FJCJ5TwGRQmVxjM7eAXr6B7JdFRGRrFFApFAZ/ppaS4+KSD5TQKQwNB/THgWEiOQxBUQKQ/MxtWgcQkTymAIihYoi9SBERNIKCDMrMbOC8PXbzOxDZhbLbNWyZ9+U3+pBiEj+SrcH8TSQMLNa4Ang08DdmapUtg0tGrRLiwaJSB5LNyDM3TuBPwP+1d0/DCzIXLWyqygeoawwSpOWHRWRPJZ2QJjZ2cAngF+HZRldbCjbasoKFRAiktfSDYjrgC8DD4TLhp4A/HfmqpV9CggRyXdp9QLc/SngKYBwsLrZ3T+XyYplW01ZIesbWrNdDRGRrEn3LqZ7zazczEqADcBGM/tiZquWXVPKEjS2dme7GiIiWZPuJaYF7t4KXA6sAGYBV2WsVhNATVkhHb0DdPT0Z7sqIiJZkW5AxMLfPVwO/Mrd+wDPXLWyb0pZcKtrc7vGIUQkP6UbELcC24AS4GkzOx7I6Qv0NWFANGqgWkTyVLqD1DcBNyUVvW5mF2amShPDlPIgIHQnk4jkq3QHqSvM7LtmVhc+vkPQm8hZNaUKCBHJb+leYroLaAM+Gj5agR9nqlITwaTiONECo7FNdzKJSH5K99fQc939iqTtr5vZqkxUaKIoKDCqSwtpbFUPQkTyU7o9iC4zO3dow8zOAboyU6WJY2pFgrf0WwgRyVPpBsQy4GYz22Zm24AfAn99qJPM7BIz22hmm83shhT7TzSzF8ysx8y+MGLfNjNba2arzKwuzXqOq+kVCXbsVUCISH5K9y6m1cApZlYebrea2XXAmtHOMbMIcDNwMVAPrDSz5e6+Iemw3cDnCH5fkcqF7t6cTh0zYVpFEU/9sQl3x8yyVQ0RkawY04py7t4a/qIa4POHOPwMYLO7b3H3XuB+YMmIv9fo7iuBCbl02/TKBJ29A7R26dfUIpJ/jmTJ0UN9pa4Ftidt14dl6XLgMTN72cyWjloJs6VDt982NTWN4c8f2rSKIgAa9ub8cIuIyAGOJCAONdVGqgAZy/Qc57j76cClwGfN7PyUlXC/zd0Xu/vimpqaMfz5Q5tWmQBghwJCRPLQQccgzKyN1B/qBhQd4m/XAzOTtmcADelWzN0bwudGM3uA4JLV0+mePx6mD/UgWjRQLSL556A9CHcvc/fyFI8ydz/UAPdKYL6ZzTGzOHAlsDydSplZiZmVDb0G3gusS+fc8VRTVki0wNSDEJG8lLFlQ92938yuBR4FIsBd4Wp0y8L9t5jZcUAdUA4MhndGLQCqgQfCO4eiwL3u/kim6jqaSIExtTzBDvUgRCQPZXRdaXdfQbB+RHLZLUmv3yK49DRSK3BKJuuWruMqEhqkFpG8dCSD1HlhemURb7YoIEQk/yggDmFOVTFv7umit38w21URETmqFBCHMKemhEGHN3Z3ZLsqIiJHlQLiEOZUlwKwpUkBISL5RQFxCCfUBOsibW5qz3JNRESOLgXEIZQnYsyYVMT6hpxegltE5AAKiDQsmlHB2vq92a6GiMhRpYBIw8m1lbyxu5OWzt5sV0VE5KhRQKRh0YwKANa+qV6EiOQPBUQaTpoeBMQaXWYSkTyigEhDRXGM2VXFrN7eku2qiIgcNQqINC2ePZmV23YzODiWJS1ERI5dCog0nXVCFXs6+/hjY1u2qyIiclQoINJ05pzJALz42q4s10RE5OhQQKRp5uRiaiuLeHHL7mxXRUTkqFBAjMHZc6t4Ycsu+gY0s6uI5D4FxBi8b+Fx7O3q47nNzdmuiohIxikgxuD8t1VTlojy0Ood2a6KiEjGKSDGoDAa4ZKFx/HY+rfo7hvIdnVERDJKATFGHz6tlraefn69Rr0IEcltCogxOntuFfOmlHL389tw14/mRCR3KSDGyMz4yz+Zzdo39/Ly63uyXR0RkYxRQByGPzutlsklcX7wxKZsV0VEJGMUEIehpDDKsnefwDObmnlpq344JyK5SQFxmK46azZTygr5xsPr6dcP50QkBykgDlNRPMLff3Ah695s5cfPbct2dURExp0C4ghcdvJxXPSOKXzn8Y28sasz29URERlXCogjYGZ88/KTiBUU8Ln7/0Bvvy41iUjuyGhAmNklZrbRzDab2Q0p9p9oZi+YWY+ZfWEs504U0yqKuPEji1i1vYV/efTVbFdHRGTcZCwgzCwC3AxcCiwAPm5mC0Ycthv4HPDtwzh3wrjs5Glcddbx3P7MVpavbsh2dURExkUmexBnAJvdfYu79wL3A0uSD3D3RndfCfSN9dyJ5ivvfwdnzJ7MF36xmhe0qJCI5IBMBkQtsD1puz4sG9dzzWypmdWZWV1TU9NhVXQ8JGIRbvvkO5lVVczSn9ax8S0tTSoix7ZMBoSlKEt38qK0z3X329x9sbsvrqmpSbtymVBZHOfuT7+LoliET9zxOzbtVEiIyLErkwFRD8xM2p4BpHuB/kjOzaoZk4q593+eiRl8/PYX1ZMQkWNWJgNiJTDfzOaYWRy4Elh+FM7NunlTyrh/6VkUmPHx219k1faWbFdJRGTMMhYQ7t4PXAs8CrwC/MLd15vZMjNbBmBmx5lZPfB54KtmVm9m5aOdm6m6ZsLcmlL+/a/PpqQwwpW3vcDjG3Zmu0oiImNiubSmweLFi72uri7b1dhPU1sP19yzkrVv7uUfPrSQT549O9tVEhEZZmYvu/viVPv0S+oMqykr5L6lZ/GeE6fytV+t58u/XKPlSkXkmKCAOAqK41FuveqdfPbCudz30nY+csvzbN+tuZtEZGJTQBwlkQLji+87kds/uZjXd3XygX99lidf1biEiExcCoij7OIFU3n4f53L9Moi/uruOv7uwXV09eqSk4hMPAqILDi+qoQHPvMnXH3uHH764uu8/6ZnWK1bYUVkglFAZEkiFuHvPrCAn19zJl19A1zxb8/z7Uc3agBbRCYMBUSWnTOvmkeuO58PnTqdH/73Zi75/tM8t7k529USEVFATAQVRTG++9FT+dnVZ+LAJ+74HZ//xSp2tfdku2oikscUEBPIufOrefS68/nshXNZvqqBC7/9W+58dqtWqhORrFBATDCJWIQvvu9EVvzteZwys5JvPryBS77/NE+8spNc+tW7iEx8CogJ6m1Ty/jJX53BXZ9aDAZX31PHX9z5O038JyJHjQJiAjMz3nPiVB697nz+/oMLeGVHG5ff/BzX3FPHhobWbFdPRHKcJus7hrT39HP3c1u59ekttHX38/6Tp3HdRfOZP7Us21UTkWPUwSbrU0Acg/Z29nHHs1u469mtdPYNcPE7prLsgrmcPmtStqsmIscYBUSO2t3Ry93PbeWeF15nb1cfZ86ZzLIL5nLB22owS7Vqq4jI/hQQOa6jp5/7XnqDO5/dyo693bxtaimfPHs2Hz6tlpLCaLarJyITmAIiT/T2D7J8dQM/fm4r6xtaKSuM8pHFM7jqrOM5oaY029UTkQlIAZFn3J3fv9HCT17Yxoq1O+gbcM6bX83H3jWTixdMpTAayXYVRWSCUEDksaa2Hu5/6Q3ufekNduztpqIoxpJTp/M/3jmTk2rLNVYhkucUEMLAoPPc5mZ+UbedxzbspLd/kBOPK+OK02fw/kXTmF5ZlO0qikgWKCBkP3s7+1i+poH/rNvO6vq9ACw+fhIfWDSNyxZNY0pZIss1FJGjRQEho9ra3MGv1zTw0OodbNzZRoHBmXOquGzRNC5+x1SOq1BYiOQyBYSkZdPONh5as4OH1zSwpakDgJNrK7joHVO5aMEUFkzTmIVIrlFAyJi4O5sb2/nNK4385pWd/P6NPbjD9IoEFy2YygVvr+HMOVX6jYVIDlBAyBFpbu/hyVcb+c2GnTyzqZmuvgFiEeP0WZM4b341582v4aTaCiIF6l2IHGsUEDJuuvsGqNu2h2c2N/HspmbWh7PKVhTFOGdeFefMq+aM2ZOZN6VUl6NEjgEHCwhdI5AxScQinDu/mnPnV8OlQe/iuc3NPLOpmWc3NbNi7VsATC6Js/j4SZwxZzLvmj2ZhdPLiUY0u7zIsUQBIUekurSQJafWsuTUWtydbbs6Wbl1Ny9t283Kbbt5bMNOAIrjEU6fNYnFsydxysxKTplRyeSSeJZrLyIHk9FLTGZ2CfADIALc4e7/NGK/hfsvAzqBT7n778N924A2YADoH60LlEyXmCaena3dvLQ1CIuXtu5m4842hv6TmzW5OAyLCk6dWcnC6RUUxTUNiMjRlJVLTGYWAW4GLgbqgZVmttzdNyQddikwP3ycCfxb+DzkQndvzlQdJfOmlif44CnT+eAp04Fg0aO19XtZXd/C6u0tvLxtNw+tbgAgUmC8fWoZC6eX845p5SwInyuKYtlsgkjeyuQlpjOAze6+BcDM7geWAMkBsQT4iQfdmBfNrNLMprn7jgzWS7KotDDK2XOrOHtu1XBZY2s3q+v3snp7C6vrW3jy1Ub+4+X64f21lUVBYEwrGw6NmZOKKdBdUyIZlcmAqAW2J23Xs3/vYLRjaoEdgAOPmZkDt7r7banexMyWAksBZs2aNT41l6NqSnmCixckuHjBVCD4HUZTWw8bdrTyyo628LmVJ1/dyWB4eaooFmHulBLm1ZQyf2oZc2tKmT+1lOMnF2swXGScZDIgUn29GzngcbBjznH3BjObAjxuZq+6+9MHHBwEx20QjEEcSYVlYjAzppQnmFKe4IK3Txku7+4bYONbbbyyo5VNje1samxn5bY9PLiqYfiYWMSYU13C/CllzJ1SypzqYo6vKmFOVQmVxTHdeisyBpkMiHpgZtL2DKAh3WPcfei50cweILhkdUBASP5IxCLBoPbMyv3K23v6ea2xnc1haGxubGN9w15WrNtB8j0Y5Ykos6tLmF1VwuyqIDiC7WIml8QVHiIjZDIgVgLzzWwO8CZwJfDnI45ZDlwbjk+cCex19x1mVgIUuHtb+Pq9wDcyWFc5hpUWRlMGR0//ANt3d7KtuZNtuzp4fVfw/Ifte3h4TcPw5SqAssIotZOKmDGpiBmTiqmtTHo9qYhJ6n1IHspYQLh7v5ldCzxKcJvrXe6+3syWhftvAVYQ3OK6meA210+Hp08FHgj/h4wC97r7I5mqq+SmwmiEeVPKmDel7IB9vf2DbN/Tyeu7OtjWHDzX7+mifk8XL27ZTXtP/37HF8cjw6FRO6mI2spiplcmmFqe4LjyBMdVJEjEdIuu5BZNtSEygrvT2tVPfUsn9Xu6eDMMjjeHtlu6aOnsO+C8iqIY0yr2hcbUiqHwKGRqeYJpFeqJyMSjqTZExsDMqCiOUVFcwcLpFSmPae/p56293exs7WZH+PzW3m7eag1eb9jRSnN7DyO/f8UiRnVpYfiIU1NWOLw99LqmLE51aSEVRQoTyS4FhMhhKC2MMm9KKfOmlI56TN/AIE1tPUFoDIdHD83twaMxvJV3V3sv/YMH9uTjkQKqSuNJ4RFnUkmcycVxJhWHr0tiTCqOM7kkTnkipt+GyLhSQIhkSCxSwPTKokOu9z046Ozt6qOpvYfmtp7gub2XprbkMOlmQ0Mruzt76e0fTPl3Cgwqi+NMKo4xuSQMkRFBMqk4HvSOioJHeSJGIlagnoqkpIAQybKCAmNSSfBB/rapBw6oJ3N3uvoG2N3Ry56OPnZ39rKno5fdHb20dPaG233s7ujljd2drNrewp7OXvoGRh9rjEcKKC+KUVEU3RccRftCZGi7PJFUFoZMSTyicMlhCgiRY4iZURyPUhyPMmNSeue4Ox29AzHIPEYAAAcuSURBVMNBsrerj9buPvZ27Xu0dvXR2tXP3q4+mtt7ea2pY/i4g93HUmDB5bayRIzSwiiliejwc1lhdL+yskSU0sJY8Dy0P9xXEo/q8tgEpIAQyXFmFnxQF0aZObl4TOcODjrtvf3s7UwKkqRwaevup627n/aeftrD55auPrbv6Rze7uwdSOu9SpMCpaQwSkk8QnE8QnE8SklhJAzGVNsRSgqT9sUjFBdGKY5FFDpHSAEhIqMqKDDKE8HlpZmHPjyl/oFBOnoHkkLkwGBpG7Gvo3eAzp5+Wjr76Ozdt93ZN3DQHs1IiVgBJfEoxYWR4Dm+f7AkYsGjKB6hKBY8ErGC/cqGj9mvLDimMJrb4zcKCBHJqGikgIqignGZtt3d6e4bpKO3n86egeC5dyAIkZ7gOfX2AB09+/Y1tfXQ1TdAd9/A8PPBxmlGY8ZwiAwFR1E8QiIahMlwsCSHShgshdF9IVMYi5AInw+1L1pgRy2UFBAicswws+BbfDwCo99hfFj6Bwbp7h+kq3f/4OjqHRgRJoP7lQ2Vd/UO7rfd3hMEUffQOWH5aHehpavA2Bce0SB4ppQl+MWys8fpX2IfBYSICEFPpzRSQGlhZj8W3Z2e/sHwMUBPX/Dc3Ze8PaKsPwif0fYVZWiaFwWEiMhRZGbD4xowsVdL1MoqIiKSkgJCRERSUkCIiEhKCggREUlJASEiIikpIEREJCUFhIiIpKSAEBGRlHJqTWozawJeP8zTq4HmcazOsUBtzg9qc3443DYf7+41qXbkVEAcCTOrG23h7lylNucHtTk/ZKLNusQkIiIpKSBERCQlBcQ+t2W7AlmgNucHtTk/jHubNQYhIiIpqQchIiIpKSBERCSlvA8IM7vEzDaa2WYzuyHb9RkvZnaXmTWa2bqksslm9riZbQqfJyXt+3L4b7DRzN6XnVofGTObaWb/bWavmNl6M/vbsDxn221mCTN7ycxWh23+elies20eYmYRM/uDmT0cbud0m81sm5mtNbNVZlYXlmW2ze6etw8gArwGnADEgdXAgmzXa5zadj5wOrAuqeyfgRvC1zcAN4avF4RtLwTmhP8mkWy34TDaPA04PXxdBvwxbFvOthswoDR8HQN+B5yVy21OavvngXuBh8PtnG4zsA2oHlGW0Tbnew/iDGCzu29x917gfmBJlus0Ltz9aWD3iOIlwD3h63uAy5PK73f3HnffCmwm+Lc5prj7Dnf/ffi6DXgFqCWH2+2B9nAzFj6cHG4zgJnNAN4P3JFUnNNtHkVG25zvAVELbE/arg/LctVUd98BwYcpMCUsz7l/BzObDZxG8I06p9sdXmpZBTQCj7t7zrcZ+D7wv4HBpLJcb7MDj5nZy2a2NCzLaJujR1DZXGApyvLxvt+c+ncws1Lg/wHXuXurWarmBYemKDvm2u3uA8CpZlYJPGBmJx3k8GO+zWb2AaDR3V82swvSOSVF2THV5tA57t5gZlOAx83s1YMcOy5tzvceRD0wM2l7BtCQpbocDTvNbBpA+NwYlufMv4OZxQjC4efu/suwOOfbDeDuLcBvgUvI7TafA3zIzLYRXBZ+j5n9jNxuM+7eED43Ag8QXDLKaJvzPSBWAvPNbI6ZxYErgeVZrlMmLQf+Mnz9l8CvksqvNLNCM5sDzAdeykL9jogFXYU7gVfc/btJu3K23WZWE/YcMLMi4CLgVXK4ze7+ZXef4e6zCf6ffdLd/4IcbrOZlZhZ2dBr4L3AOjLd5myPzGf7AVxGcLfLa8BXsl2fcWzXfcAOoI/g28TVQBXwBLApfJ6cdPxXwn+DjcCl2a7/Ybb5XIJu9BpgVfi4LJfbDSwC/hC2eR3wtbA8Z9s8ov0XsO8uppxtM8GdlqvDx/qhz6pMt1lTbYiISEr5folJRERGoYAQEZGUFBAiIpKSAkJERFJSQIiISEoKCJExMLOBcDbNoce4zQBsZrOTZ98VybZ8n2pDZKy63P3UbFdC5GhQD0JkHIRz9d8Yrs3wkpnNC8uPN7MnzGxN+DwrLJ9qZg+E6zisNrM/Cf9UxMxuD9d2eCz8dbRIViggRMamaMQlpo8l7Wt19zOAHxLMNkr4+ifuvgj4OXBTWH4T8JS7n0Kwbsf6sHw+cLO7LwRagCsy3B6RUemX1CJjYGbt7l6aonwb8B533xJOGPiWu1eZWTMwzd37wvId7l5tZk3ADHfvSfobswmm654fbn8JiLn7tzLfMpEDqQchMn58lNejHZNKT9LrATROKFmkgBAZPx9Len4hfP08wYyjAJ8Ang1fPwH8DQwv+FN+tCopki59OxEZm6Jw9bYhj7j70K2uhWb2O4IvXh8Pyz4H3GVmXwSagE+H5X8L3GZmVxP0FP6GYPZdkQlDYxAi4yAcg1js7s3ZrovIeNElJhERSUk9CBERSUk9CBERSUkBISIiKSkgREQkJQWEiIikpIAQEZGU/j8yk82nm1xJsQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAdfUlEQVR4nO3de5hcdZ3n8feHTpM7JIQQIJ2kI8YhRC4LTcjKRWbiCAgOsoMCuhodFHBBdLwsjPLAsoyOjjoz8gCDgXGAXSSCgCKiwIKgcg8QIE0IuRCgCZck3eHSnU53ur/7xzkdik4l6SR9+lTX+byep56qc+pU1fdXT9Kf+p3fOb+jiMDMzIprp7wLMDOzfDkIzMwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yCwqibpPkktkoZm8N6SdK6khZJaJTVJuknS/v39WWZZchBY1ZJUDxwJBPA3GXzET4CvAucCuwEfAH4FHL+tbyRpSP+WZtZ3DgKrZp8DHgauAeaUPiFpkqRbJK2StEbSZSXPfUnSIklvS3pW0sG931jSNOBs4LSIuDci1kdEW0RcHxHfT7e5T9IXS17zeUl/LlkOSWdLWgIskXSlpB/1+pxfS/p6+nhvSTenNb8g6dx++I7MHARW1T4HXJ/ejpE0AUBSDXA78CJQD0wE5qXPfRL4X+lrdyHpSawp896zgaaIeHQHa/wEcBiwH/Bz4BRJSmsZC3wUmCdpJ+A3wFNpvbOBr0k6Zgc/38xBYNVJ0hHAFODGiHgcWAZ8On16JrA38K2IaI2I9ojo+aX+ReCfI+KxSCyNiBfLfMQ44NV+KPWfIqI5ItYBfyLZjXVk+tzJwEMRsRI4FBgfEf87IjoiYjlwFXBqP9RgBecgsGo1B7grIlanyz/n3d1Dk4AXI2JDmddNIgmNrVkD7LXDVcLLPQ8imQFyHnBauurTJL0ZSEJtb0lre27At4EJ/VCDFZwHqKzqSBoOfAqokfRaunooMEbSgSR/fCdLGlImDF4G9unDx9wDXC6pISLmb2abVmBEyfKeZbbpPf3vDcBdkr5PssvopJK6XoiIaX2ozWybuEdg1egTQBfJfveD0tt0kl0vnwMeJdmt831JIyUNk3R4+tqrgW9KOiQ9PPT9kqb0/oCIWAJcAdwg6WhJO6fvc6qk89PNFgD/TdIISe8HTt9a4RHxJLAqrePOiFibPvUo8Jak8yQNl1Qj6YOSDt2eL8islIPAqtEc4D8j4qWIeK3nBlwGfAYQ8HHg/cBLQBNwCkBE3AR8l2RX0tskh4PutpnPOTd9z8uBtSS7lE4iGdQF+FegA3gduJZ3d/NszQ3AR9IaSOvqSms+CHgBWE0SFrv28T3NNku+MI2ZWbG5R2BmVnAOAjOzgnMQmJkVnIPAzKzgBt15BLvvvnvU19fnXYaZ2aDy+OOPr46I8eWeG3RBUF9fz/z5mzt/x8zMypFUbqoUwLuGzMwKz0FgZlZwDgIzs4JzEJiZFZyDwMys4DILAkk/k/SGpIWbeV6SLpW0VNLT5S4HaGZm2cuyR3ANcOwWnj8OmJbezgD+PcNazMxsMzI7jyAi/iipfgubnAhcl16V6WFJYyTtFRH9cfk/Sz3+Ygv3L34j7zLMrB801O/GUR8oe07YDsnzhLKJlFymj2RO+ImUuQ6spDNIeg1Mnjx5QIqrFj+88zkeXt5Mcjl0MxvMzvrwPlUXBOX+NJW9OEJEzAXmAjQ0NPgCCtugubWDY2ZM4Kefbci7FDOrUHkeNdREcqHwHnXAypxqqVrNrZ3sNnLnvMswswqWZxDcBnwuPXpoFvCmxwf6V0TQ0tbB2BEOAjPbvMx2DUm6ATga2F1SE3ARUAsQEVcCdwAfA5YCbcAXsqqlqN5q30BXd7hHYGZblOVRQ6dt5fkAzs7q8w3WtnUAuEdgZlvkM4urWHNrEgTuEZjZlgy66xFYee2dXfzL3c/zdvuGjetee3MdAGNG1OZVlpkNAg6CKrHg5bXM/eNyxoyopbbm3Y7etD1G8b7xo3KszMwqnYOgSrSku4Fu+NIspu+1S87VmNlg4jGCKtHc5vEAM9s+DoIq0dMj8HiAmW0rB0GVaGnrZOTONQwdUpN3KWY2yDgIqkRLawdjvVvIzLaDg6BKNLd1eHzAzLaLjxqqcA8uW81/PrCC2MqcqwteXssBdWMGpigzqyoOggp3yxOvcP/iVbx/jy2fC7D3rsP52Af3HKCqzKyaOAgqXEtrB9MmjOK35x6ZdylmVqU8RlDhvO/fzLLmIKhwLa2+noCZZctBUOGaW90jMLNsOQgq2Iaubt5q3+Czhc0sUw6CCrZ2XSfg+YPMLFs+aqhC3PjYy1z/6EvvWbe+swvwFcbMLFsOggpx21MrWbG6lYMmlZwUNryW+nEjOWzqbvkVZmZVz0FQIZpbOzi0fixXzzk071LMrGA8RlAhWto6GONdQGaWAwdBhWjxiWNmlhMHQQVY19FFe2e3B4XNLBcOggrw7mUmfb6AmQ08B0EFePcyk+4RmNnAcxDk7K7G1/i7ax4DfL6AmeXDQZCzB5etYe26Tr5weD0H1O2adzlmVkA+jyBnLW0d7LXrMC76+Iy8SzGzgnKPIGfNnmbazHLmIMjZ2rZOxnp2UTPLkYMgZ82tHYz1iWRmliMHQc5a2jrYzbuGzCxHDoIctXd20dbR5R6BmeXKQZCjtW3JhWc8WGxmeXIQ5Ki51VNLmFn+HAQ5aknnGHKPwMzy5CDI0bs9AgeBmeXHQZCjnh6BJ5szszw5CHLU0poMFo/xCWVmlqNMg0DSsZIWS1oq6fwyz+8q6TeSnpLUKOkLWdZTaVraOthl2BBqa5zHZpafzP4CSaoBLgeOA/YDTpO0X6/NzgaejYgDgaOBH0sqzH6S5lZfntLM8pflT9GZwNKIWB4RHcA84MRe2wQwWpKAUUAzsCHDmiqKL1hvZpUgyyCYCLxcstyUrit1GTAdWAk8A3w1Irp7v5GkMyTNlzR/1apVWdU74HzBejOrBFkGgcqsi17LxwALgL2Bg4DLJO2yyYsi5kZEQ0Q0jB8/vv8rzUlLa6fPITCz3GUZBE3ApJLlOpJf/qW+ANwSiaXAC8C+GdZUUZIxAh8xZGb5yjIIHgOmSZqaDgCfCtzWa5uXgNkAkiYAfwEsz7CmitHe2cW6zi6PEZhZ7jK7VGVEbJB0DnAnUAP8LCIaJZ2VPn8lcAlwjaRnSHYlnRcRq7OqqZL0nEzmMQIzy1um1yyOiDuAO3qtu7Lk8Urgo1nWUKl6ppfwGIGZ5c1nMuWgqztY9OrbgHsEZpa/THsEVt5P7lnCpfcsAWCP0UNzrsbMis49ghy8tKaV3UcN5f+cPpP63UfmXY6ZFZyDIAfNbZ1MHDOMI6dVzzkRZjZ4OQhysLatw9cpNrOK4SDIQXNrh48WMrOK4SDIQYuDwMwqiINggLV3dtHa0eWpJcysYvjw0QHw+lvtLF/VCsCb69ITyTxGYGYVwkEwAE6/9jEWvvLWe9btvevwnKoxM3svB8EAWLm2nb/ebwJ/d/hUAIbV7sSBdWNyrsrMLOEgyFh3d7C2rYN99xzNf91nXN7lmJltwoPFGXurvZPu8ORyZla5HAQZ2zjLqI8SMrMK5SDIWEtbJ+AegZlVLgdBxlpafQEaM6tsDoKM3fXsa4B7BGZWuRwEGXq7vZMb5zcBsPsoX3fAzCqTgyBDa95JdgtdcPx0hu9ck3M1ZmblOQgy1JxeoH6fPUblXImZ2eY5CDK0caDY4wNmVsEcBBlq9hFDZjYIOAgytDY9h2DMCJ9MZmaVy0GQoea2DmprxKihntLJzCqXgyAjHRu6+fkjLzFmxM5IyrscM7PNchBk5MFlq3lzXacHis2s4jkIMrI6PYfgys8eknMlZmZb5iDISM+ho+NGuUdgZpXNQZCR5rYOhuwkRnug2MwqnIMgI2vbOhg70gPFZlb5HAQZaW7t8ECxmQ0KDoKMtLR2+kQyMxsUvAO7H73V3sl//OkF1m/oZtmqd5g5dbe8SzIz2yoHQT+6b/EqfnLPEnau2QkEh0wZm3dJZmZb5SDoR83vrAfg4W/P9kRzZjZoeIygHzW3dSLBrsM9NmBmg4eDoB+1tHYwZngtNTv5kFEzGzwcBP2oua3DF6k3s0En0yCQdKykxZKWSjp/M9scLWmBpEZJ92dZT9Z6TiIzMxtMtjpYLGkksC4iutPlnYBhEdG2ldfVAJcDfw00AY9Jui0ini3ZZgxwBXBsRLwkaY/tb0r+mls7mThmeN5lmJltk74cNXQP8BHgnXR5BHAX8KGtvG4msDQilgNImgecCDxbss2ngVsi4iWAiHij76VXhp/ev4znX0++mhfXtLL/xF1yrsjMbNv0JQiGRURPCBAR70ga0YfXTQReLlluAg7rtc0HgFpJ9wGjgZ9ExHW930jSGcAZAJMnT+7DRw+Mzq5u/ul3z7HLsCGMHlbLbiN35ohp4/Muy8xsm/QlCFolHRwRTwBIOgRY14fXlTt0Jsp8/iHAbGA48JCkhyPi+fe8KGIuMBegoaGh93vkpqUtmWr6W8fuy2dnTcm5GjOz7dOXIPgacJOklenyXsApfXhdEzCpZLkOWFlmm9UR0UoSOH8EDgSeZxBoaU0uTu/J5cxsMNtqEETEY5L2Bf6C5Ff+cxHR2Yf3fgyYJmkq8ApwKsmYQKlfA5dJGgLsTLLr6F+3of5c9fQIxnpyOTMbxLZ6+Kiks4GREbEwIp4BRkn6H1t7XURsAM4B7gQWATdGRKOksySdlW6zCPg98DTwKHB1RCzc/uYMrJ6rkPmQUTMbzPqya+hLEXF5z0JEtEj6Eslhn1sUEXcAd/Rad2Wv5R8CP+xbuZWlOe0ReF4hMxvM+nJC2U4qucxWen6A//IBa9uSPWS+7oCZDWZ96RHcCdwo6UqSo37OAn6XaVWDwDvrN/DDOxczvLaGoUNq8i7HzGy79SUIziM5hv/LJIPFT5IcOVRoC195E4ADJ+2acyVmZjtmq7uG0qklHgaWAw0kx/wvyriuitczUHzhCTNyrsTMbMdstkcg6QMkh3yeBqwBfgEQEX85MKVVtpZ0fMADxWY22G1p19BzwJ+Aj0fEUgBJfz8gVQ0CPecQeKDYzAa7Le0a+lvgNeAPkq6SNJvy00YUUnNrByN2rmFYrQeKzWxw22wQRMStEXEKsC9wH/D3wARJ/y7powNUX8Vq8UVozKxK9GWwuDUiro+IE0jmC1oAlL3ITJG0tHZ4fMDMqkJfDh/dKCKagZ+mt0Hv8Rebufg3z7Kha9snNF2xppVDpozNoCozs4G1TUFQbf74/GqebnqTj0yfsM2v3XvMcE4+pC6DqszMBlahg6ClrYNdh9dy9ZyGvEsxM8tNphevr3QtbZ3ez29mhVfsIGjt8LUEzKzwCh0EzT7yx8ys2EHQ0tbBGJ8LYGYFV+ggcI/AzKzAQbCuo4v1G7o9V5CZFV5hg2D9hi4AhnuuIDMruMIGQXd6MvFO8jx6ZlZsBQ6CJAl2cg6YWcEVPgjkHoGZFVxhgyC8a8jMDChwEHjXkJlZosBBkNy7R2BmRVfcIOjuGSPIuRAzs5wVNgg8RmBmlihsELx71FDOhZiZ5azwQeAegZkVXYGDILl3DphZ0RU2CMI9AjMzoMBB4MNHzcwSBQ4Cn1BmZgYOAs81ZGaFV9ggePc8gnzrMDPLm4PAPQIzK7jCBsHGMYLCfgNmZonC/hn0GIGZWSLTIJB0rKTFkpZKOn8L2x0qqUvSyVnWU8qHj5qZJTILAkk1wOXAccB+wGmS9tvMdj8A7syqlnLCh4+amQHZ9ghmAksjYnlEdADzgBPLbPcV4GbgjQxr2YR7BGZmiSyDYCLwcslyU7puI0kTgZOAKzOsoyzPPmpmlsgyCMr9iY1ey/8GnBcRXVt8I+kMSfMlzV+1alW/FOfZR83MEkMyfO8mYFLJch2wstc2DcC89Mid3YGPSdoQEb8q3Sgi5gJzARoaGnqHyXbxeQRmZoksg+AxYJqkqcArwKnAp0s3iIipPY8lXQPc3jsEsuK5hszMEpkFQURskHQOydFANcDPIqJR0lnp8wM+LlDq3esROAnMrNiy7BEQEXcAd/RaVzYAIuLzWdbSm3sEZmaJwp5Z7AvTmJklChsE3d3JvYPAzIquuEHg8wjMzIBCB0Fy7x6BmRVdYYMgPA21mRlQ4CBwj8DMLFHgIPDho2Zm4CDwCWVmVniFDQLPNWRmlihsEHjXkJlZosBBkNy7R2BmRVfgIPAJZWZmUOAg8FxDZmaJwgaBdw2ZmSUKHAQeLDYzg0IHQXLv8wjMrOgKGwThHoGZGVDgIOju9pnFZmZQ5CDYOFicbx1mZnkrcBC4R2BmBgUOgnCPwMwMKHAQdPuEMjMzoNBBkNw7CMys6AocBJ5ryMwMChwEnmvIzCxR2CDw4aNmZokCB4F7BGZmUOggSO6dA2ZWdIUNgohA8gllZmaFDYLuCO8WMjOj0EHggWIzMyh0EIR3C5mZUeAgCPcIzMwAGJJ3AXnp7vYYgVml6OzspKmpifb29rxLGfSGDRtGXV0dtbW1fX5NYYMg8DkEZpWiqamJ0aNHU19f7122OyAiWLNmDU1NTUydOrXPryvsrqHu9PBRM8tfe3s748aNcwjsIEmMGzdum3tWhQ2CZIzA/+jMKoVDoH9sz/dY2CBIziPIuwozs/wVPAicBGZmmQaBpGMlLZa0VNL5ZZ7/jKSn09uDkg7Msp5S3eGuqJkZZBgEkmqAy4HjgP2A0yTt12uzF4APR8QBwCXA3Kzq6S28a8jMNuOcc85hypQpeZcxYLLsEcwElkbE8ojoAOYBJ5ZuEBEPRkRLuvgwUJdhPe/R3e3BYjPb1AsvvMB9991HR0cHb7/9dmaf09XVldl7b6sszyOYCLxcstwEHLaF7U8HflfuCUlnAGcATJ48uV+K82CxWWW6+DeNPLvyrX59z/323oWLPj6jT9tedNFFXHDBBVx11VU0NjYya9YsAFauXMlXvvIVli9fzrp167juuuuoq6vbZN3MmTOZNWsW8+bNo76+nldeeYUTTzyR+fPn88lPfpJJkybx5JNPMnv2bPbdd19+9KMfsW7dOkaPHs2tt97K+PHjy37W8OHDOeuss3jggQcAeOKJJ/jmN7/Jvffeu8PfT5ZBUO7PbJTdUPpLkiA4otzzETGXdLdRQ0ND2ffYVh4jMLPeGhsbWbhwIddeey1//vOfNwbBhg0bOO644/jud7/LCSecQFtbG11dXRxxxBGbrIsIXnrppY27lp5++mn2339/AJ555hmmT5/OH/7wBwDWrFnDySefDMDFF1/MjTfeyJlnnln2s0aOHMmyZcvo6uqipqaGb3zjG/z4xz/ul3ZnGQRNwKSS5TpgZe+NJB0AXA0cFxFrMqznPSKCnQp7zJRZ5errL/csfOc73+GSSy5BEtOnT2fhwoUA/OpXv2L69OmccMIJAIwYMYJf/vKXm6wDWLJkCVOnTt34Q7MnCNrb22lububCCy/c+HnXXHMNv/jFL1i/fj2vvfYa3/ve98p+Vo8ZM2bQ2NjIkiVLmDx5MgcffHC/tDvLIHgMmCZpKvAKcCrw6dINJE0GbgE+GxHPZ1jLJnz4qJmVeuSRR7jzzjtZsGABZ599Nu3t7RxwwAEALFiwYOMuoh7l1kHyq7+nBwAwf/58zjzzTBobGznssMMYMiT5s3vdddfx6KOPcu+99zJq1CiOOuooZsyYwe233172fQFmzZrFAw88wBVXXMHvf//7/mp6doPFEbEBOAe4E1gE3BgRjZLOknRWutmFwDjgCkkLJM3Pqp7eun1msZmV+Pa3v83tt9/OihUrWLFiBU899dTGHsGee+5JY2Pjxm1XrVpVdh1Ac3Mzw4cPB2DRokX89re/Zf/99+eZZ57ZGCyQBMaHPvQhRo0axc0338yDDz7I/vvvv9n3hSQILrjgAk466SQmTpzYb23PdOdIRNwRER+IiH0i4rvpuisj4sr08RcjYmxEHJTeGrKsp5TnGjKzHnfffTfr169n9uzZG9dNmDCB1tZWmpub+fznP8/rr7/OjBkzOOigg3jooYfKrgM45phjuOeee/jUpz7FTTfdxLhx45gwYcImQTBnzhwuvfRSjjzySJ5//nne9773MXLkyM2+L8C+++7L0KFDOe+88/q1/Yrol7HXAdPQ0BDz5+94x+Hs659g8etv8/++/uF+qMrMdsSiRYuYPn163mVUvHPOOYdDDz2UOXPmbHG7ct+npMc392O7MNNQ3//8Kv7x9mc3Lr/6Zjt77Tosx4rMzPpm2bJlHH/88Rx++OFbDYHtUZggGDV0CNMmjNq4PG3CKI6aNj7HiszM+mafffbhueeey+z9CxMEh0wZyyFTDsm7DDOziuMj6c3MCs5BYGYVYbAduFKptud7dBCYWe6GDRvGmjVrHAY7qOeaxcOGbduBMIUZIzCzylVXV0dTU9N7Tp6y7TNs2DDq6rZtImcHgZnlrra2lqlTp+ZdRmF515CZWcE5CMzMCs5BYGZWcINuriFJq4AXt/PluwOr+7GcwcBtLga3uRh2pM1TIqLsdAqDLgh2hKT5AznDaSVwm4vBbS6GrNrsXUNmZgXnIDAzK7iiBcHcvAvIgdtcDG5zMWTS5kKNEZiZ2aaK1iMwM7NeHARmZgVXmCCQdKykxZKWSjo/73r6i6SfSXpD0sKSdbtJulvSkvR+bMlz/5B+B4slHZNP1TtG0iRJf5C0SFKjpK+m66u23ZKGSXpU0lNpmy9O11dtmwEk1Uh6UtLt6XJVtxdA0gpJz0haIGl+ui7bdkdE1d+AGmAZ8D5gZ+ApYL+86+qnth0FHAwsLFn3z8D56ePzgR+kj/dL2z4UmJp+JzV5t2E72rwXcHD6eDTwfNq2qm03IGBU+rgWeASYVc1tTtvxdeDnwO3pclW3N23LCmD3XusybXdRegQzgaURsTwiOoB5wIk519QvIuKPQHOv1ScC16aPrwU+UbJ+XkSsj4gXgKUk382gEhGvRsQT6eO3gUXARKq43ZF4J12sTW9BFbdZUh1wPHB1yeqqbe9WZNruogTBRODlkuWmdF21mhARr0LyRxPYI11fdd+DpHrgv5D8Qq7qdqe7SRYAbwB3R0S1t/nfgP8JdJesq+b29gjgLkmPSzojXZdpu4tyPQKVWVfE42ar6nuQNAq4GfhaRLwllWtesmmZdYOu3RHRBRwkaQxwq6QPbmHzQd1mSScAb0TE45KO7stLyqwbNO3t5fCIWClpD+BuSc9tYdt+aXdRegRNwKSS5TpgZU61DITXJe0FkN6/ka6vmu9BUi1JCFwfEbekq6u+3QARsRa4DziW6m3z4cDfSFpBsiv3ryT9X6q3vRtFxMr0/g3gVpJdPZm2uyhB8BgwTdJUSTsDpwK35VxTlm4D5qSP5wC/Lll/qqShkqYC04BHc6hvhyj56f8fwKKI+JeSp6q23ZLGpz0BJA0HPgI8R5W2OSL+ISLqIqKe5P/rvRHx36nS9vaQNFLS6J7HwEeBhWTd7rxHyAdwJP5jJEeXLAO+k3c9/diuG4BXgU6SXwenA+OAe4Al6f1uJdt/J/0OFgPH5V3/drb5CJLu79PAgvT2sWpuN3AA8GTa5oXAhen6qm1zSTuO5t2jhqq6vSRHNj6V3hp7/lZl3W5PMWFmVnBF2TVkZmab4SAwMys4B4GZWcE5CMzMCs5BYGZWcA4Cs14kdaUzP/bc+m22Wkn1pTPFmlWCokwxYbYt1kXEQXkXYTZQ3CMw66N0nvgfpNcFeFTS+9P1UyTdI+np9H5yun6CpFvTawg8JelD6VvVSLoqva7AXemZwma5cRCYbWp4r11Dp5Q891ZEzAQuI5kdk/TxdRFxAHA9cGm6/lLg/og4kOSaEY3p+mnA5RExA1gL/G3G7THbIp9ZbNaLpHciYlSZ9SuAv4qI5emkd69FxDhJq4G9IqIzXf9qROwuaRVQFxHrS96jnmQK6Wnp8nlAbUT8Y/YtMyvPPQKzbRObeby5bcpZX/K4C4/VWc4cBGbb5pSS+4fSxw+SzJAJ8Bngz+nje4Avw8aLyuwyUEWabQv/EjHb1PD0SmA9fh8RPYeQDpX0CMmPqNPSdecCP5P0LWAV8IV0/VeBuZJOJ/nl/2WSmWLNKorHCMz6KB0jaIiI1XnXYtafvGvIzKzg3CMwMys49wjMzArOQWBmVnAOAjOzgnMQmJkVnIPAzKzg/j/b1jgUtzzi1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 利用鸢尾花数据集，实现前向传播、反向传播，可视化loss曲线\n",
    "\n",
    "# 导入所需模块\n",
    "import tensorflow as tf\n",
    "from sklearn import datasets\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import time  ##1##\n",
    "\n",
    "# 导入数据，分别为输入特征和标签\n",
    "x_data = datasets.load_iris().data\n",
    "y_data = datasets.load_iris().target\n",
    "\n",
    "# 随机打乱数据（因为原始数据是顺序的，顺序不打乱会影响准确率）\n",
    "# seed: 随机数种子，是一个整数，当设置之后，每次生成的随机数都一样（为方便教学，以保每位同学结果一致）\n",
    "np.random.seed(116)  # 使用相同的seed，保证输入特征和标签一一对应\n",
    "np.random.shuffle(x_data)\n",
    "np.random.seed(116)\n",
    "np.random.shuffle(y_data)\n",
    "tf.random.set_seed(116)\n",
    "\n",
    "# 将打乱后的数据集分割为训练集和测试集，训练集为前120行，测试集为后30行\n",
    "x_train = x_data[:-30]\n",
    "y_train = y_data[:-30]\n",
    "x_test = x_data[-30:]\n",
    "y_test = y_data[-30:]\n",
    "\n",
    "# 转换x的数据类型，否则后面矩阵相乘时会因数据类型不一致报错\n",
    "x_train = tf.cast(x_train, tf.float32)\n",
    "x_test = tf.cast(x_test, tf.float32)\n",
    "\n",
    "# from_tensor_slices函数使输入特征和标签值一一对应。（把数据集分批次，每个批次batch组数据）\n",
    "train_db = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(32)\n",
    "test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)\n",
    "\n",
    "# 生成神经网络的参数，4个输入特征故，输入层为4个输入节点；因为3分类，故输出层为3个神经元\n",
    "# 用tf.Variable()标记参数可训练\n",
    "# 使用seed使每次生成的随机数相同（方便教学，使大家结果都一致，在现实使用时不写seed）\n",
    "w1 = tf.Variable(tf.random.truncated_normal([4, 3], stddev=0.1, seed=1))\n",
    "b1 = tf.Variable(tf.random.truncated_normal([3], stddev=0.1, seed=1))\n",
    "\n",
    "lr = 0.1  # 学习率为0.1\n",
    "train_loss_results = []  # 将每轮的loss记录在此列表中，为后续画loss曲线提供数据\n",
    "test_acc = []  # 将每轮的acc记录在此列表中，为后续画acc曲线提供数据\n",
    "epoch = 500  # 循环500轮\n",
    "loss_all = 0  # 每轮分4个step，loss_all记录四个step生成的4个loss的和\n",
    "\n",
    "##########################################################################\n",
    "m_w, m_b = 0, 0\n",
    "beta = 0.9\n",
    "##########################################################################\n",
    "\n",
    "# 训练部分\n",
    "now_time = time.time()  ##2##\n",
    "for epoch in range(epoch):  # 数据集级别的循环，每个epoch循环一次数据集\n",
    "    for step, (x_train, y_train) in enumerate(train_db):  # batch级别的循环 ，每个step循环一个batch\n",
    "        with tf.GradientTape() as tape:  # with结构记录梯度信息\n",
    "            y = tf.matmul(x_train, w1) + b1  # 神经网络乘加运算\n",
    "            y = tf.nn.softmax(y)  # 使输出y符合概率分布（此操作后与独热码同量级，可相减求loss）\n",
    "            y_ = tf.one_hot(y_train, depth=3)  # 将标签值转换为独热码格式，方便计算loss和accuracy\n",
    "            loss = tf.reduce_mean(tf.square(y_ - y))  # 采用均方误差损失函数mse = mean(sum(y-out)^2)\n",
    "            loss_all += loss.numpy()  # 将每个step计算出的loss累加，为后续求loss平均值提供数据，这样计算的loss更准确\n",
    "        # 计算loss对各个参数的梯度\n",
    "        grads = tape.gradient(loss, [w1, b1])\n",
    "\n",
    "        ##########################################################################\n",
    "        # sgd-momentun  \n",
    "        m_w = beta * m_w + (1 - beta) * grads[0]\n",
    "        m_b = beta * m_b + (1 - beta) * grads[1]\n",
    "        w1.assign_sub(lr * m_w)\n",
    "        b1.assign_sub(lr * m_b)\n",
    "    ##########################################################################\n",
    "\n",
    "    # 每个epoch，打印loss信息\n",
    "    print(\"Epoch {}, loss: {}\".format(epoch, loss_all / 4))\n",
    "    train_loss_results.append(loss_all / 4)  # 将4个step的loss求平均记录在此变量中\n",
    "    loss_all = 0  # loss_all归零，为记录下一个epoch的loss做准备\n",
    "\n",
    "    # 测试部分\n",
    "    # total_correct为预测对的样本个数, total_number为测试的总样本数，将这两个变量都初始化为0\n",
    "    total_correct, total_number = 0, 0\n",
    "    for x_test, y_test in test_db:\n",
    "        # 使用更新后的参数进行预测\n",
    "        y = tf.matmul(x_test, w1) + b1\n",
    "        y = tf.nn.softmax(y)\n",
    "        pred = tf.argmax(y, axis=1)  # 返回y中最大值的索引，即预测的分类\n",
    "        # 将pred转换为y_test的数据类型\n",
    "        pred = tf.cast(pred, dtype=y_test.dtype)\n",
    "        # 若分类正确，则correct=1，否则为0，将bool型的结果转换为int型\n",
    "        correct = tf.cast(tf.equal(pred, y_test), dtype=tf.int32)\n",
    "        # 将每个batch的correct数加起来\n",
    "        correct = tf.reduce_sum(correct)\n",
    "        # 将所有batch中的correct数加起来\n",
    "        total_correct += int(correct)\n",
    "        # total_number为测试的总样本数，也就是x_test的行数，shape[0]返回变量的行数\n",
    "        total_number += x_test.shape[0]\n",
    "    # 总的准确率等于total_correct/total_number\n",
    "    acc = total_correct / total_number\n",
    "    test_acc.append(acc)\n",
    "    print(\"Test_acc:\", acc)\n",
    "    print(\"--------------------------\")\n",
    "total_time = time.time() - now_time  ##3##\n",
    "print(\"total_time\", total_time)  ##4##\n",
    "\n",
    "# 绘制 loss 曲线\n",
    "plt.title('Loss Function Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Loss')  # y轴变量名称\n",
    "plt.plot(train_loss_results, label=\"$Loss$\")  # 逐点画出trian_loss_results值并连线，连线图标是Loss\n",
    "plt.legend()  # 画出曲线图标\n",
    "plt.show()  # 画出图像\n",
    "\n",
    "# 绘制 Accuracy 曲线\n",
    "plt.title('Acc Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Acc')  # y轴变量名称\n",
    "plt.plot(test_acc, label=\"$Accuracy$\")  # 逐点画出test_acc值并连线，连线图标是Accuracy\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# 请将loss曲线、ACC曲线、total_time记录到 class2\\优化器对比.docx  对比各优化器收敛情况\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adagrad算法-在SGD基础增加了二阶动量\n",
    "$$ m_t = g_t $$\n",
    "$$ V_t = Σ_{\\tau = 1}^t g_{\\tau}^2 $$\n",
    "$$ \\eta _t =\\frac{lr*m_t}{\\sqrt{V_t}} = \\frac{lr*g_t}{\\sqrt{Σ_{\\tau = 1}^t g_{\\tau}^2}} $$\n",
    "$$ w_{t+1} = w_t - \\eta _t = w_t - \\frac{lr*g_t}{\\sqrt{Σ_{\\tau = 1}^t g_{\\tau}^2}} $$\n",
    "        \n",
    "        v_w, v_b = 0, 0  \n",
    "        v_w += tf.square(grads[0])  \n",
    "        v_b += tf.square(grads[1])  \n",
    "        w1.assign_sub(lr * grads[0] / tf.sqrt(v_w))  \n",
    "        b1.assign_sub(lr * grads[1] / tf.sqrt(v_b))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, loss: 0.2528788633644581\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 1, loss: 0.17821525409817696\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 2, loss: 0.15621443092823029\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 3, loss: 0.1420477293431759\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 4, loss: 0.13362513668835163\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 5, loss: 0.1275476086884737\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 6, loss: 0.12286805734038353\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 7, loss: 0.11908590793609619\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 8, loss: 0.11591362580657005\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 9, loss: 0.11317718587815762\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 10, loss: 0.11076570674777031\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 11, loss: 0.1086051631718874\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 12, loss: 0.10664406791329384\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 13, loss: 0.10484533570706844\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 14, loss: 0.10318147204816341\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 15, loss: 0.10163157060742378\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 16, loss: 0.10017935931682587\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 17, loss: 0.09881196171045303\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 18, loss: 0.09751898981630802\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 19, loss: 0.09629195556044579\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 20, loss: 0.09512384235858917\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 21, loss: 0.09400876797735691\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 22, loss: 0.09294173866510391\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 23, loss: 0.09191850759088993\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 24, loss: 0.09093539975583553\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 25, loss: 0.08998921886086464\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 26, loss: 0.08907720074057579\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 27, loss: 0.08819687366485596\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 28, loss: 0.08734607137739658\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 29, loss: 0.08652284927666187\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 30, loss: 0.08572548255324364\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 31, loss: 0.08495240099728107\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 32, loss: 0.08420220948755741\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 33, loss: 0.08347361162304878\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 34, loss: 0.08276545815169811\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 35, loss: 0.08207667618989944\n",
      "Test_acc: 0.6666666666666666\n",
      "--------------------------\n",
      "Epoch 36, loss: 0.08140628971159458\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 37, loss: 0.0807534009218216\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 38, loss: 0.08011719025671482\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 39, loss: 0.07949688285589218\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 40, loss: 0.07889177277684212\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 41, loss: 0.0783011969178915\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 42, loss: 0.07772453129291534\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 43, loss: 0.07716120593249798\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 44, loss: 0.07661068439483643\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 45, loss: 0.0760724600404501\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 46, loss: 0.07554607093334198\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 47, loss: 0.07503105141222477\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 48, loss: 0.07452699262648821\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 49, loss: 0.07403350155800581\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 50, loss: 0.07355018984526396\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 51, loss: 0.07307672034949064\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 52, loss: 0.07261274475604296\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 53, loss: 0.07215794175863266\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 54, loss: 0.07171202544122934\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 55, loss: 0.07127469405531883\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 56, loss: 0.07084567565470934\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 57, loss: 0.07042471412569284\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 58, loss: 0.07001155707985163\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 59, loss: 0.06960596982389688\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 60, loss: 0.06920771952718496\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 61, loss: 0.06881659384816885\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 62, loss: 0.06843238975852728\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 63, loss: 0.06805489677935839\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 64, loss: 0.06768393237143755\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 65, loss: 0.06731931306421757\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 66, loss: 0.06696086190640926\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 67, loss: 0.06660841591656208\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 68, loss: 0.06626179907470942\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 69, loss: 0.06592087540775537\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 70, loss: 0.06558548379689455\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 71, loss: 0.06525548174977303\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 72, loss: 0.06493072677403688\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 73, loss: 0.06461109966039658\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 74, loss: 0.06429645605385303\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 75, loss: 0.06398667115718126\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 76, loss: 0.0636816406622529\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 77, loss: 0.06338123884052038\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 78, loss: 0.06308535020798445\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 79, loss: 0.06279388070106506\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 80, loss: 0.06250670831650496\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 81, loss: 0.062223742716014385\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 82, loss: 0.06194488238543272\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 83, loss: 0.061670041643083096\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 84, loss: 0.06139912083745003\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 85, loss: 0.06113203428685665\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 86, loss: 0.060868692584335804\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 87, loss: 0.06060901936143637\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 88, loss: 0.06035292614251375\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 89, loss: 0.06010034400969744\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 90, loss: 0.05985118914395571\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 91, loss: 0.059605385176837444\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 92, loss: 0.05936287995427847\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 93, loss: 0.05912358220666647\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 94, loss: 0.058887443505227566\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 95, loss: 0.058654384687542915\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 96, loss: 0.05842434335500002\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 97, loss: 0.05819726828485727\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 98, loss: 0.05797308683395386\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 99, loss: 0.05775175150483847\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 100, loss: 0.057533202692866325\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 101, loss: 0.05731738731265068\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 102, loss: 0.057104241102933884\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 103, loss: 0.056893715634942055\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 104, loss: 0.056685774587094784\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 105, loss: 0.056480354629457\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 106, loss: 0.05627741012722254\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 107, loss: 0.0560768973082304\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 108, loss: 0.055878762155771255\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 109, loss: 0.05568297579884529\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 110, loss: 0.055489485152065754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 111, loss: 0.05529824271798134\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 112, loss: 0.05510922335088253\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 113, loss: 0.054922361858189106\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 114, loss: 0.054737649857997894\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 115, loss: 0.05455502774566412\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 116, loss: 0.05437447130680084\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 117, loss: 0.05419592093676329\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 118, loss: 0.05401937384158373\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 119, loss: 0.05384476948529482\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 120, loss: 0.05367208365350962\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 121, loss: 0.053501286543905735\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 122, loss: 0.05333234276622534\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 123, loss: 0.05316521506756544\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 124, loss: 0.052999887615442276\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 125, loss: 0.052836320362985134\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 126, loss: 0.052674476988613605\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 127, loss: 0.0525143351405859\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 128, loss: 0.05235587619245052\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 129, loss: 0.05219906009733677\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 130, loss: 0.052043866366147995\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 131, loss: 0.05189026799052954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 132, loss: 0.05173823423683643\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 133, loss: 0.051587752997875214\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 134, loss: 0.05143878236413002\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 135, loss: 0.051291316747665405\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 136, loss: 0.05114530585706234\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 137, loss: 0.05100075714290142\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 138, loss: 0.05085763242095709\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 139, loss: 0.050715912133455276\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 140, loss: 0.05057556740939617\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 141, loss: 0.05043659079819918\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 142, loss: 0.050298950634896755\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 143, loss: 0.05016263574361801\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 144, loss: 0.050027623772621155\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 145, loss: 0.04989389330148697\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 146, loss: 0.04976141732186079\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 147, loss: 0.049630191177129745\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 148, loss: 0.049500186927616596\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 149, loss: 0.04937139060348272\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 150, loss: 0.04924378264695406\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 151, loss: 0.04911735188215971\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 152, loss: 0.048992074094712734\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 153, loss: 0.04886793531477451\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 154, loss: 0.04874491784721613\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 155, loss: 0.04862301051616669\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 156, loss: 0.04850219655781984\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 157, loss: 0.04838245268911123\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 158, loss: 0.04826377797871828\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 159, loss: 0.04814614076167345\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 160, loss: 0.048029541969299316\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 161, loss: 0.04791396111249924\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 162, loss: 0.04779937490820885\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 163, loss: 0.04768578801304102\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 164, loss: 0.04757317528128624\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 165, loss: 0.04746152553707361\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 166, loss: 0.04735082387924194\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 167, loss: 0.04724106006324291\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 168, loss: 0.047132219187915325\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 169, loss: 0.04702429939061403\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 170, loss: 0.04691727552562952\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 171, loss: 0.04681114759296179\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 172, loss: 0.04670589230954647\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 173, loss: 0.046601501293480396\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 174, loss: 0.04649797081947327\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 175, loss: 0.046395277604460716\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 176, loss: 0.04629342444241047\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 177, loss: 0.046192388981580734\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 178, loss: 0.04609216935932636\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 179, loss: 0.045992753468453884\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 180, loss: 0.04589412175118923\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 181, loss: 0.045796277932822704\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 182, loss: 0.04569920618087053\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 183, loss: 0.0456028962507844\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 184, loss: 0.045507339760661125\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 185, loss: 0.045412528328597546\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 186, loss: 0.04531845357269049\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 187, loss: 0.04522509593516588\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 188, loss: 0.045132461935281754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 189, loss: 0.04504053946584463\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 190, loss: 0.04494931176304817\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 191, loss: 0.044858778826892376\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 192, loss: 0.0447689238935709\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 193, loss: 0.04467975068837404\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 194, loss: 0.04459124058485031\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 195, loss: 0.04450339451432228\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 196, loss: 0.04441619571298361\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 197, loss: 0.04432963766157627\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 198, loss: 0.04424372315406799\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 199, loss: 0.04415843263268471\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 200, loss: 0.04407376889139414\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 201, loss: 0.043989721685647964\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 202, loss: 0.04390626959502697\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 203, loss: 0.043823424726724625\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 204, loss: 0.04374117869883776\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 205, loss: 0.04365951381623745\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 206, loss: 0.0435784375295043\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 207, loss: 0.04349792469292879\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 208, loss: 0.043417991138994694\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 209, loss: 0.04333861265331507\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 210, loss: 0.04325979296118021\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 211, loss: 0.04318152088671923\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 212, loss: 0.04310378897935152\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 213, loss: 0.04302659910172224\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 214, loss: 0.04294993914663792\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 215, loss: 0.0428738035261631\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 216, loss: 0.04279818665236235\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 217, loss: 0.042723086662590504\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 218, loss: 0.04264849238097668\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 219, loss: 0.04257440287619829\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 220, loss: 0.0425008125603199\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 221, loss: 0.042427713982760906\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 222, loss: 0.04235510341823101\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 223, loss: 0.04228297248482704\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 224, loss: 0.04221132770180702\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 225, loss: 0.04214014112949371\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 226, loss: 0.042069438844919205\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 227, loss: 0.041999188251793385\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 228, loss: 0.04192939028143883\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 229, loss: 0.041860051453113556\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 230, loss: 0.0417911633849144\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 231, loss: 0.04172272142022848\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 232, loss: 0.04165470972657204\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 233, loss: 0.04158714320510626\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 234, loss: 0.04151999671012163\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 235, loss: 0.041453284211456776\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 236, loss: 0.04138698894530535\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 237, loss: 0.04132111184298992\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 238, loss: 0.0412556529045105\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 239, loss: 0.04119059629738331\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 240, loss: 0.04112594947218895\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 241, loss: 0.04106170404702425\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 242, loss: 0.04099785536527634\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 243, loss: 0.040934400632977486\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 244, loss: 0.04087133426219225\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 245, loss: 0.04080865252763033\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 246, loss: 0.040746355429291725\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 247, loss: 0.04068444482982159\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 248, loss: 0.04062290117144585\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 249, loss: 0.04056172166019678\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 250, loss: 0.040500921197235584\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 251, loss: 0.040440477430820465\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 252, loss: 0.040380396880209446\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 253, loss: 0.04032067349180579\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 254, loss: 0.04026130912825465\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 255, loss: 0.04020228469744325\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 256, loss: 0.04014361882582307\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 257, loss: 0.04008529148995876\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 258, loss: 0.04002730268985033\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 259, loss: 0.039969656616449356\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 260, loss: 0.039912338834255934\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 261, loss: 0.03985536238178611\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 262, loss: 0.0397987044416368\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 263, loss: 0.039742381777614355\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 264, loss: 0.03968637203797698\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 265, loss: 0.03963069012388587\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 266, loss: 0.039575318805873394\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 267, loss: 0.03952026180922985\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 268, loss: 0.03946551866829395\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 269, loss: 0.03941108426079154\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 270, loss: 0.039356952998787165\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 271, loss: 0.03930312441661954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 272, loss: 0.03924960130825639\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 273, loss: 0.039196368772536516\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 274, loss: 0.03914343658834696\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 275, loss: 0.03909078938886523\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 276, loss: 0.039038438349962234\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 277, loss: 0.03898637695237994\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 278, loss: 0.038934596348553896\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 279, loss: 0.03888309095054865\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 280, loss: 0.03883188217878342\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 281, loss: 0.038780934643000364\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 282, loss: 0.038730265107005835\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 283, loss: 0.03867987310513854\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 284, loss: 0.03862974978983402\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 285, loss: 0.038579890970140696\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 286, loss: 0.03853030316531658\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 287, loss: 0.038480975199490786\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 288, loss: 0.03843190986663103\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 289, loss: 0.038383102510124445\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 290, loss: 0.038334549870342016\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 291, loss: 0.03828626358881593\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 292, loss: 0.038238216657191515\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 293, loss: 0.03819042583927512\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 294, loss: 0.03814287669956684\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 295, loss: 0.038095582742244005\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 296, loss: 0.03804852953180671\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 297, loss: 0.03800171799957752\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 298, loss: 0.0379551500082016\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 299, loss: 0.03790881624445319\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 300, loss: 0.03786272695288062\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 301, loss: 0.037816866766661406\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 302, loss: 0.03777123475447297\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 303, loss: 0.0377258388325572\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 304, loss: 0.037680668756365776\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 305, loss: 0.037635731510818005\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 306, loss: 0.03759101312607527\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 307, loss: 0.037546522449702024\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 308, loss: 0.03750225296244025\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 309, loss: 0.037458199076354504\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 310, loss: 0.03741436451673508\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 311, loss: 0.03737075021490455\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 312, loss: 0.037327347323298454\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 313, loss: 0.03728415770456195\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 314, loss: 0.03724117996171117\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 315, loss: 0.03719841269776225\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 316, loss: 0.037155854515731335\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 317, loss: 0.03711350075900555\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 318, loss: 0.03707135282456875\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 319, loss: 0.03702940698713064\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 320, loss: 0.036987668834626675\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 321, loss: 0.03694612393155694\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 322, loss: 0.036904781591147184\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 323, loss: 0.036863632034510374\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 324, loss: 0.03682268084958196\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 325, loss: 0.03678192524239421\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 326, loss: 0.03674135822802782\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 327, loss: 0.036700986325740814\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 328, loss: 0.036660803481936455\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 329, loss: 0.03662080969661474\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 330, loss: 0.036581003572791815\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 331, loss: 0.0365413841791451\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 332, loss: 0.03650194592773914\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 333, loss: 0.036462686490267515\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 334, loss: 0.03642361657693982\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 335, loss: 0.03638472454622388\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 336, loss: 0.03634601039811969\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 337, loss: 0.036307476460933685\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 338, loss: 0.03626911295577884\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 339, loss: 0.036230926401913166\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 340, loss: 0.03619291167706251\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 341, loss: 0.03615507436916232\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 342, loss: 0.03611740609630942\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 343, loss: 0.03607991011813283\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 344, loss: 0.03604257805272937\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 345, loss: 0.03600541688501835\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 346, loss: 0.03596842149272561\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 347, loss: 0.03593159047886729\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 348, loss: 0.03589492104947567\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 349, loss: 0.035858419723808765\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 350, loss: 0.03582207439467311\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 351, loss: 0.03578589158132672\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 352, loss: 0.03574986895546317\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 353, loss: 0.03571400558575988\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 354, loss: 0.03567830054089427\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 355, loss: 0.03564274683594704\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 356, loss: 0.03560735424980521\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 357, loss: 0.0355721078813076\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 358, loss: 0.03553701797500253\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 359, loss: 0.03550208080559969\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 360, loss: 0.035467295441776514\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 361, loss: 0.035432655829936266\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 362, loss: 0.035398168954998255\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 363, loss: 0.035363827366381884\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 364, loss: 0.03532963106408715\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 365, loss: 0.035295580979436636\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 366, loss: 0.03526167990639806\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 367, loss: 0.03522791713476181\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 368, loss: 0.0351942996494472\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 369, loss: 0.03516082093119621\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 370, loss: 0.035127485636621714\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 371, loss: 0.03509429330006242\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 372, loss: 0.035061230417340994\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 373, loss: 0.03502831142395735\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 374, loss: 0.03499553399160504\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 375, loss: 0.034962881822139025\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 376, loss: 0.03493037261068821\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 377, loss: 0.034897992853075266\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 378, loss: 0.0348657495342195\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 379, loss: 0.03483364265412092\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 380, loss: 0.034801659639924765\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 381, loss: 0.034769812133163214\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 382, loss: 0.03473809454590082\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 383, loss: 0.034706505946815014\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 384, loss: 0.03467504447326064\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 385, loss: 0.034643708262592554\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 386, loss: 0.03461250429973006\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 387, loss: 0.03458141954615712\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 388, loss: 0.03455046145245433\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 389, loss: 0.034519628155976534\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 390, loss: 0.034488916862756014\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 391, loss: 0.034458329901099205\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 392, loss: 0.03442787192761898\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 393, loss: 0.034397524781525135\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 394, loss: 0.03436729637905955\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 395, loss: 0.03433719417080283\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 396, loss: 0.034307208843529224\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 397, loss: 0.03427734086290002\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 398, loss: 0.03424758883193135\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 399, loss: 0.03421795554459095\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 400, loss: 0.03418843587860465\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 401, loss: 0.034159034956246614\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 402, loss: 0.03412974439561367\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 403, loss: 0.034100569784641266\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 404, loss: 0.0340715073980391\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 405, loss: 0.03404255723580718\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 406, loss: 0.03401371743530035\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 407, loss: 0.033984990790486336\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 408, loss: 0.033956370782107115\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 409, loss: 0.03392786579206586\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 410, loss: 0.03389946557581425\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 411, loss: 0.033871171064674854\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 412, loss: 0.03384298598393798\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 413, loss: 0.03381491079926491\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 414, loss: 0.033786938060075045\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 415, loss: 0.033759071957319975\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 416, loss: 0.033731311559677124\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 417, loss: 0.033703653141856194\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 418, loss: 0.033676101826131344\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 419, loss: 0.03364865016192198\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 420, loss: 0.033621301874518394\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 421, loss: 0.03359405091032386\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 422, loss: 0.0335669107735157\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 423, loss: 0.03353986516594887\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 424, loss: 0.033512923400849104\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 425, loss: 0.03348607383668423\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 426, loss: 0.03345932811498642\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 427, loss: 0.03343268111348152\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 428, loss: 0.033406130969524384\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 429, loss: 0.03337967349216342\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 430, loss: 0.033353318460285664\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 431, loss: 0.033327057026326656\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 432, loss: 0.03330089105293155\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 433, loss: 0.03327482147142291\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 434, loss: 0.03324884083122015\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 435, loss: 0.03322295472025871\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 436, loss: 0.033197169192135334\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 437, loss: 0.033171471673995256\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 438, loss: 0.03314586589112878\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 439, loss: 0.03312035324051976\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 440, loss: 0.033094930462539196\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 441, loss: 0.03306959290057421\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 442, loss: 0.03304435545578599\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 443, loss: 0.033019198570400476\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 444, loss: 0.032994134817272425\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 445, loss: 0.0329691618680954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 446, loss: 0.032944271340966225\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 447, loss: 0.03291947115212679\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 448, loss: 0.03289476176723838\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 449, loss: 0.032870129216462374\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 450, loss: 0.03284559166058898\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 451, loss: 0.032821131870150566\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 452, loss: 0.032796763349324465\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 453, loss: 0.032772475853562355\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 454, loss: 0.03274827403947711\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 455, loss: 0.032724153250455856\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 456, loss: 0.03270012000575662\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 457, loss: 0.032676166389137506\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 458, loss: 0.03265230171382427\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 459, loss: 0.03262850735336542\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 460, loss: 0.032604798674583435\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 461, loss: 0.032581175211817026\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 462, loss: 0.03255762765184045\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 463, loss: 0.032534160651266575\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 464, loss: 0.032510772813111544\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 465, loss: 0.032487460412085056\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 466, loss: 0.03246423415839672\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 467, loss: 0.03244108520448208\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 468, loss: 0.03241801122203469\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 469, loss: 0.03239501640200615\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 470, loss: 0.032372101210057735\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 471, loss: 0.03234925540164113\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 472, loss: 0.03232648828998208\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 473, loss: 0.0323037994094193\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 474, loss: 0.03228118596598506\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 475, loss: 0.03225864330306649\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 476, loss: 0.03223617793992162\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 477, loss: 0.032213787548244\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 478, loss: 0.03219147538766265\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 479, loss: 0.03216922888532281\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 480, loss: 0.03214705968275666\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 481, loss: 0.0321249570697546\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 482, loss: 0.03210293175652623\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 483, loss: 0.03208098001778126\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 484, loss: 0.03205909579992294\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 485, loss: 0.032037283293902874\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 486, loss: 0.03201554296538234\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 487, loss: 0.03199387062340975\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 488, loss: 0.03197226766496897\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 489, loss: 0.03195073688402772\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 490, loss: 0.03192927362397313\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 491, loss: 0.03190788719803095\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 492, loss: 0.0318865617737174\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 493, loss: 0.031865301076322794\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 494, loss: 0.031844109296798706\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 495, loss: 0.031822992488741875\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 496, loss: 0.031801935750991106\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 497, loss: 0.03178095072507858\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 498, loss: 0.03176002483814955\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 499, loss: 0.03173916973173618\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "total_time 10.871759176254272\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3RdZZ3/8ff3nJOTe9I2SW9JS1soMG0BgYogF0HRAW/VYUYrDKP+wA4Koy5xfuKM4290nFnDjFcUh5uVm8DoaBGciiAIiFBpCoXeKL3bNG3TpG3u93x/f+yd5DQ9aZM0pyc9+bzWOuvs/ey9c56na/V8zvPsvZ9t7o6IiMhAkXRXQERExiYFhIiIJKWAEBGRpBQQIiKSlAJCRESSUkCIiEhSCgiRNDCzJjObk+56iByJAkLSxsy2m9nlafjce82sI/yS7n19NIWf96yZXZ9Y5u4F7r41RZ93tZlVhu3abWa/NrOLUvFZktkUEDJe/Uf4Jd37+u90V2g0mNkXgO8C/wZMAWYCPwQWjeBvxUa3dnKiUUDImGNm2Wb2XTOrDl/fNbPscFupmf3KzA6a2X4z+72ZRcJtXzKzXWbWaGYbzexdw/zce83sGwnrl5pZVcL6djP7opm9bmb1ZvbfZpaTsH2Rma02swYz22JmV5jZvwIXAz8If9H/INzXzeyUcLnYzO43s31mtsPMvpLQpk+Y2Qtm9k0zO2Bm28zsykHqXwx8HbjR3X/h7s3u3unuj7v73w+jjV8ys9eB5rAu/zPgc75nZrcl1P1HYU9ll5l9w8yiw/l3l7FLvxBkLPpH4HzgLYADvwS+AvwTcDNQBZSF+54PuJmdBtwEvNXdq81sFpCKL6qPAFcAbcAfgE8Ad5jZecD9wF8CTwPTgEJ3f8LMLgQedPd7Bvmb3weKgTlACfAksBv4Ubj9bcB9QCmwBPiRmZX74fPkXADkAMuOsY0fA94H1AKTgX8wsyJ3bwi//D8CfDjc9z5gL3AKkA/8CtgJ3HmMdZAxQD0IGYuuAb7u7jXuvg/4GnBtuK2T4Mv3pPDX8e/DL8puIBuYZ2ZZ7r7d3bcc4TO+GPZCDppZ7TDqdpu7V7v7fuBxghADuA5Y6u5PuXuPu+9y9zeO9sfCL9yPAl9290Z33w58K6G9ADvc/W537yb4Qp5GMHw0UAlQ6+5dw2hPMre5+053b3X3HcArwIfCbe8EWtx9hZlNAa4EPh/2VmqA7wCLj/HzZYxQQMhYNB3YkbC+IywD+E9gM/CkmW01s1sA3H0z8Hngn4EaM3vEzKYzuG+6+4TwVTqMuu1JWG4BCsLlGcCRAmkwpUCcw9tbnuwz3b0lXCzgcHVA6SicO9g5YP0hgl4FwNXhOsBJQBawuzdsCXoOk4/x82WMUEDIWFRN8OXTa2ZYRvgr+2Z3nwN8APhC77kGd3/I3S8Kj3Xg1mF+bjOQl7A+dRjH7gROHmTbkaZMriXoFQ1s765hfHavlwiGvj50hH2G0saB9f0ZcKmZVRAMLfUGxE6gHShNCNsid58/grrLGKSAkHTLMrOchFcMeBj4ipmVmVkp8FXgQQAze7+ZnWJmBjQQDC11m9lpZvbO8GR2G9AabhuO1cB7zWySmU0l6JEM1Y+AT5rZu8wsYmblZnZ6uG0vwfmFw4TDRj8F/tXMCs3sJOALve0dDnevJ/i3ut3MPmRmeWaWZWZXmtl/jLSN4TDfs8CPgW3uviEs301wvuRbZlYUtvtkM3vHcOsuY5MCQtJtOcGXee/rn4FvAJXA68AagjHw3itv5gK/BZoIfjH/0N2fJTj/8O8Ev8j3EJ5cHWZdHgBeA7YTfPEN+dJXd38Z+CTBGHw98Bz9vYLvAX8ZXoV0W5LD/47gl/1W4AWCX+hLh1n33np8myBgvgLsI/iVfxPwaLjLSNv4EHA5/b2HXn9DMES2HjgA/A/BORLJAKYHBomISDLqQYiISFIKCBERSUoBISIiSSkgREQkqYyaaqO0tNRnzZqV7mqIiJwwVq1aVevuZcm2ZVRAzJo1i8rKynRXQ0TkhGFmOwbbpiEmERFJSgEhIiJJKSBERCSpjDoHISIyHJ2dnVRVVdHW1pbuqqRcTk4OFRUVZGVlDfkYBYSIjFtVVVUUFhYya9YsgvkfM5O7U1dXR1VVFbNnzx7ycRpiEpFxq62tjZKSkowOBwAzo6SkZNg9JQWEiIxrmR4OvUbSTgUEcNvTm3juzX3proaIyJiS0oAwsyvMbKOZbe59NOSA7deY2evh60UzOyth23YzW2Nmq80spXe//dezW/jD5uE8llhEJPOl7CR1+DD224F3A1XASjN7zN3XJ+y2DXiHux8wsyuBu4C3JWy/zN1T/s0dMejp0XMxREQSpbIHcR6w2d23unsH8AiwKHEHd3/R3Q+EqyuAihTWZ1ARM7r14CQRSZM777yTG2+8Md3VOEwqA6Kc4HGHvarCssFcB/w6Yd2BJ81slZktGewgM1tiZpVmVrlv38jOI0QihvJBRNLl9ddf54wzzkh3NQ6TyoBIdso86dewmV1GEBBfSii+0N3PAa4EbjSzS5Id6+53uftCd19YVpZ0QsKjihj0KCFEJE3WrFlzWEC88cYbXHLJJcyfP5/LL7+c2tpgtP2+++7j3HPP5cwzz+Tiiy8etGw0pPJGuSpgRsJ6BVA9cCczOxO4B7jS3et6y929OnyvMbNlBENWz6eiohEzunUOQmRc+9rj61hf3TCqf3Pe9CL+3wfmH3W/tWvXsmDBgr719vZ2rrrqKh588EHOPvtsbr31Vr7zne9wyy23cOutt7J69Wri8TgHDx6ksbHxsLLRksoexEpgrpnNNrM4sBh4LHEHM5sJ/AK41t3fTCjPN7PC3mXgPcDaVFU0EjGUDyKSDjt37qSwsJDi4uK+skcffZSLLrqIs88+G4B58+ZRU1NDNBqltbWVm2++mcrKSiZMmJC0bLSkrAfh7l1mdhPwGyAKLHX3dWZ2Q7j9DuCrQAnww/Amji53XwhMAZaFZTHgIXd/IlV1jVhwK7qIjF9D+aWfCsnOP6xfv/6QsjVr1jBv3jzy8vJYu3Ytjz/+OEuWLOH666/nM5/5TNKy0ZDSuZjcfTmwfEDZHQnL1wPXJzluK3DWwPJU0RCTiKRLsvMP5eXlrF69GoCtW7fywAMP8MILL7Bp0ybmzp3L4sWLWb9+PW1tbUnLRosm6yMICOWDiKTDmjVreOKJJ3j44YcBmDZtGs888wzLly/njDPOIDc3l6VLl1JSUsLNN9/MSy+9RH5+PvPnz+fuu+/mhhtuOKxstCgggEhEQ0wikh4/+clPkpY/+uijh5Xde++9QyobLZqLCd0oJyKSjAICiGqISUTkMAoIwHSjnMi4NV6Gl0fSTgUE4UlqdSFExp2cnBzq6uoyPiR6nyiXk5MzrON0khqIRkw9CJFxqKKigqqqKkY6j9uJpPeZ1MOhgCB40lJ3T7prISLHW1ZW1rCe0TzeaIgJ3UktIpKMAgINMYmIJKOAIBxiUj6IiBxCAYGGmEREklFA0HujnAJCRCSRAgLN5ioikowCgmCyPuWDiMihFBAEPQidgxAROZQCAg0xiYgko4BAz6QWEUlGAYEucxURSUYBgR4YJCKSjAKC3um+010LEZGxRQFBMMSkG+VERA6lgCDsQSggREQOoYCgdzbXdNdCRGRsUUCgZ1KLiCSjgEDPpBYRSUYBgYaYRESSUUCgISYRkWQUEGiISUQkGQUEvQ8MSnctRETGFgUEvc+DUEKIiCRSQACmG+VERA6jgEBDTCIiySgg0FxMIiLJKCAIhpj0RDkRkUOlNCDM7Aoz22hmm83sliTbrzGz18PXi2Z21lCPHU3RiKEOhIjIoVIWEGYWBW4HrgTmAR8zs3kDdtsGvMPdzwT+BbhrGMeOGg0xiYgcLpU9iPOAze6+1d07gEeARYk7uPuL7n4gXF0BVAz12NEU0RCTiMhhUhkQ5cDOhPWqsGww1wG/HuGxxySiISYRkcPEUvi3LUlZ0q9hM7uMICAuGsGxS4AlADNnzhx+LdEQk4hIMqnsQVQBMxLWK4DqgTuZ2ZnAPcAid68bzrEA7n6Xuy9094VlZWUjqmjEjG4FhIjIIVIZECuBuWY228ziwGLgscQdzGwm8AvgWnd/czjHjqaIBUNMrpAQEemTsiEmd+8ys5uA3wBRYKm7rzOzG8LtdwBfBUqAH5oZQFfYG0h6bKrqGgk+G/dg6m8REUntOQjcfTmwfEDZHQnL1wPXD/XYVImEodDtTiTp6Q8RkfFHd1ITXMUEOlEtIpJIAcGhQ0wiIhJQQJAwxKSb5URE+iggCOZiAg0xiYgkUkAQzOYK0NOT5oqIiIwhCgj6h5jUgxAR6aeAQENMIiLJKCDoH2LSdBsiIv0UEPQPMSkfRET6KSCAqGmISURkIAUE/TfK6T4IEZF+Cgj6J+hTB0JEpJ8CAl3FJCKSjAICDTGJiCSjgABi0SAguhQQIiJ9FBBAdiwKQHun5toQEemlgACyY8E/Q3tXd5prIiIydiggSAwI9SBERHopIIDsrHCIST0IEZE+Cgj6exAd6kGIiPRRQKAhJhGRZBQQJAwx6SomEZE+Cgh0FZOISDIKCDTEJCKSjAKChBvlFBAiIn0UEEBW1DCD9k4NMYmI9FJAEDxyNDsWUQ9CRCSBAiIUjyogREQSKSBC2VlRXcUkIpJAARHKjkV0H4SISAIFREjnIEREDqWACGXHNMQkIpJIARHKzlIPQkQkkQIilB2L0Kb7IERE+iggQnnxGC0dCggRkV4KiFB+tgJCRCRRSgPCzK4ws41mttnMbkmy/XQze8nM2s3siwO2bTezNWa22swqU1lPgPx4lOb2rlR/jIjICSOWqj9sZlHgduDdQBWw0swec/f1CbvtBz4LfGiQP3OZu9emqo6J8rNjCggRkQRD6kGYWb6ZRcLlU83sg2aWdZTDzgM2u/tWd+8AHgEWJe7g7jXuvhLoHEHdR1V+PEpLZzc9PZ7uqoiIjAlDHWJ6Hsgxs3LgaeCTwL1HOaYc2JmwXhWWDZUDT5rZKjNbMthOZrbEzCrNrHLfvn3D+POHysuO4Q5tuhdCRAQYekCYu7cAfwF8390/DMw72jFJyobz8/xCdz8HuBK40cwuSbaTu9/l7gvdfWFZWdkw/vyh8rOD0bYmDTOJiADDCAgzuwC4BvjfsOxo5y+qgBkJ6xVA9VAr5u7V4XsNsIxgyCpl8uPBQ4Na2tWDEBGBoQfE54EvA8vcfZ2ZzQF+d5RjVgJzzWy2mcWBxcBjQ/mw8JxHYe8y8B5g7RDrOiJ58SDvmjvUgxARgSFexeTuzwHPAYQnq2vd/bNHOabLzG4CfgNEgaVhuNwQbr/DzKYClUAR0GNmnycYuioFlplZbx0fcvcnRtLAoSoIh5ia1YMQEQGGGBBm9hBwA9ANrAKKzezb7v6fRzrO3ZcDyweU3ZGwvIdg6GmgBuCsodRttORlB0NM6kGIiASGOsQ0z90bCO5XWA7MBK5NWa3SoLcHoXMQIiKBoQZEVnjfw4eAX7p7J8O7ImnM6w2Ixra035IhIjImDDUg7gS2A/nA82Z2EsEwUMaYlB8HoK65I801EREZG4Z6kvo24LaEoh1mdllqqpQeOVlR8uJRDiggRESAoU+1UWxm3+69Y9nMvkXQm8goE/Pi7FdAiIgAQx9iWgo0Ah8JXw3Aj1NVqXQpKYhriElEJDTU2VxPdverEta/ZmarU1GhdJqUH6euSQEhIgJD70G0mtlFvStmdiHQmpoqpc+kfA0xiYj0GmoP4gbgfjMrDtcPAB9PTZXSZ1JenLrmdtyd8C5uEZFxa0g9CHd/zd3PAs4EznT3s4F3prRmaTClKIe2zh4a2nQ3tYjIsB456u4N4R3VAF9IQX3SatqEHAD21LeluSYiIul3LM+kzrgxmGnFQUBU12fc6RURkWE7loDIqKk2AKYV5wKw+6B6ECIiRzxJbWaNJA8CA3JTUqM0mlyYTcRgj3oQIiJHDgh3LzxeFRkLYtEIU4tyqDqggBAROZYhpow0qzSfbXXN6a6GiEjaKSAGmFWaz/ZaBYSIiAJigDml+Rxo6eRgi+6oFpHxTQExwOzSYJLaLfua0lwTEZH0UkAMMG96EQDrqjPqeUgiIsOmgBhgalEOJflx1u6qT3dVRETSSgExgJmxoLyYNbvUgxCR8U0BkcSC8iI27W2krbM73VUREUkbBUQSC6YX09XjvLm3Md1VERFJGwVEEgvKg8devFal8xAiMn4pIJKomJjLtOIcVmypS3dVRETSRgGRhJlx4Sml/GFLLT09GTdprYjIkCggBnHRKaUcbOlk/W5dzSQi45MCYhBvP7kEgBc216a5JiIi6aGAGMTkohxOn1rIMxtq0l0VEZG0UEAcwRULprJyx35qGvSEOREZfxQQR/C+M6bhDk+s25PuqoiIHHcKiCOYO6WQUyYX8KvXd6e7KiIix50C4igWnTWdl7ftZ4eeMici44wC4ij+auEMIgaPrNyZ7qqIiBxXCoijmFqcwztPn8zPKqvo6OpJd3VERI6blAaEmV1hZhvNbLOZ3ZJk++lm9pKZtZvZF4dz7PF0zfknUdvUzmOvVaezGiIix1XKAsLMosDtwJXAPOBjZjZvwG77gc8C3xzBscfNpaeWcfrUQu58boum3hCRcSOVPYjzgM3uvtXdO4BHgEWJO7h7jbuvBDqHe+zxZGZ8+tKT2VTTxNNv6MY5ERkfUhkQ5UDimd2qsGxUjzWzJWZWaWaV+/btG1FFh+J9Z0yjYmIut/9uM+7qRYhI5ktlQFiSsqF+sw75WHe/y90XuvvCsrKyIVduuGLRCDdedgqrdx7kN+v2puxzRETGilQGRBUwI2G9AhjqWd5jOTZl/urcCuZOLuDff71BVzSJSMZLZUCsBOaa2WwziwOLgceOw7EpE4tG+If3/hnb61r4yR93pLs6IiIplbKAcPcu4CbgN8AG4Kfuvs7MbjCzGwDMbKqZVQFfAL5iZlVmVjTYsamq63BceloZF88t5dtPvcleTeInIhnMMumE68KFC72ysjLln7Ottpkrvvs8l502mTuuPTflnycikipmtsrdFybbpjupR2B2aT6fu3wuT6zbwxNrNZGfiGQmBcQIferiOSwoL+LLv1jD7vrWdFdHRGTUKSBGKCsa4bbFZ9Pe1cPnHllNt+6wFpEMo4A4BnPKCviXRQt4edt+vv/MpnRXR0RkVCkgjtFV51bwF2eX872nN/H0Bt1AJyKZQwExCr7x4QXMn17EZx9+lQ27G9JdHRGRUaGAGAV58Rj3/M1bKciJcf19ldQ06v4IETnxKSBGydTiHO75m7eyv7mDjy9dSX3LwAlqRUROLAqIUXRGRTF3XnsuW2qa+PiPX6apvSvdVRIRGTEFxCi75NQyvn/12azZVc91965USIjICUsBkQJ/Pn8q3/noW6jccYBr7l7BgeaOdFdJRGTYFBAp8sGzpnPHX5/Lhj2NfOTOl9hTrxPXInJiUUCk0LvnTeHeT76V6oOtXPVfL/LGHl0CKyInDgVEir395FIeWXIBnd09XPXDF3ly3Z50V0lEZEgUEMfBGRXFPHbTRZw8uYC/fXAVP3hmEz2au0lExjgFxHEytTiHn/7tBXzgzOl888k3+T/3raSuqT3d1RIRGZQC4jjKyYryvcVv4V8WzefFLXW897bf89KWunRXS0QkKQXEcWZmXHvBLJZ95u3kx2Ncfc8Kvv74elo6dL+EiIwtCog0mT+9mMf/7iKuPf8klv5hG1d+7/es2KrehIiMHQqINMrPjvH1RQt4+FPn4w6L71rBPyxboxvrRGRMUECMARecXMITn7+Y6y6azX+v3Mll33qWB1bs0FPqRCStFBBjRF48xj+9fx7/+9mLOH1qIf/06Fo+8P0XeGFTbbqrJiLjlAJijDl9ahEPf+p8br/6HOpbO/nrH/2Rq+9ewat/OpDuqonIOKOAGIPMjPedOY1nvvgOvvr+eWzc08iHf/gin7q/kvXVmq5DRI4Pc8+cce6FCxd6ZWVluqsx6prbu1j6wjbuen4rje1dXHZaGZ++9BTeOmsiZpbu6onICczMVrn7wqTbFBAnjvqWTh5YsZ2lf9jO/uYOzj1pIn97yRze9WdTiEYUFCIyfAqIDNPa0c3PVu3kzue2sutgK+UTcrnm/Jl8dOEMSgqy0109ETmBKCAyVFd3D0+t38v9L+3gpa11xGMR3n/GNK45fybnzNTwk4gcnQJiHNi0t5EHVuzg56uqaO7oZnZpPn9xdjkfPqeciol56a6eiIxRCohxpKm9i+VrdvPzVVX8cdt+AC6YU8JV51bwnvlTKMrJSnMNRWQsUUCMUzv3t7Ds1V38/JUqdtS1kBU1Lp5bxhULpvKeeVOYkBdPdxVFJM0UEOOcu/PqzoP8es1ulq/Zw66DrcQixgUnl3Dlgmm8688mM6UoJ93VFJE0UEBIH3dn7a4Glq/dza/X7GZ7XQsA86YVcdnpZVx22mTeMmMCsajuoRQZDxQQkpS7s3FvI797Yx+/21jDqh0H6O5xinOzuOTUMi4+pZQLTi5hxiSd5BbJVAoIGZL61k5e2FTL7zbW8OzGfdSGj0StmJjL208u4e0nB4Gh4SiRzJG2gDCzK4DvAVHgHnf/9wHbLdz+XqAF+IS7vxJu2w40At1A12ANSKSAGD3uzqaaJl7cXMtLW+tYsXU/9a2dAMwpzefckyZy7kkTOeekiZxSVkBEd3KLnJCOFBCxFH5oFLgdeDdQBaw0s8fcfX3CblcCc8PX24D/Ct97Xebumu86DcyMU6cUcuqUQj5x4Wy6e5wNuxt4aUsdf9xWx2837OVnq6oAKMqJcfbMiZwzMwiNs2YUU6jLaUVOeCkLCOA8YLO7bwUws0eARUBiQCwC7vegG7PCzCaY2TR3353CeskIRCPGgvJiFpQX86lL5uDubKtt5pU/HWTVjgO8suMA3336TdzBDGaX5nNGeTELpgfHzC8v0j0YIieYVAZEObAzYb2KQ3sHg+1TDuwGHHjSzBy4093vSmFdZZjMjDllBcwpK+Avz60AoKGtk9V/OsirfzrI2up6Xt62n1+uru47ZlZJHvPD0Jg3vYjTphQypShbU4KIjFGpDIhk/+sHnvA40j4Xunu1mU0GnjKzN9z9+cM+xGwJsARg5syZx1JfOUZFOcHVT5ecWtZXVtvUztpd9ayrbmBNVT2v7TzI/77e30Eszs3itCmFnDq1gNOmFHLa1CJOnVKgm/hExoBUBkQVMCNhvQKoHuo+7t77XmNmywiGrA4LiLBncRcEJ6lHq/IyOkoLsrn0tMlcetrkvrIDzR1s3NvIxj2NbNzbyJt7Gvnl6moa27r69plSlM3cyYXMKctndml+0FspzWf6hFxNbS5ynKQyIFYCc81sNrALWAxcPWCfx4CbwvMTbwPq3X23meUDEXdvDJffA3w9hXWV42hifpzz55Rw/pySvjJ3Z09DG2/sCQJj455GtuxrYtkru2hs7w+OeCzCrJI85pQWMLssnzml+cwpy2fGpDzKCjRcJTKaUhYQ7t5lZjcBvyG4zHWpu68zsxvC7XcAywkucd1McJnrJ8PDpwDLwv/sMeAhd38iVXWV9DMzphXnMq04l8sSehvuTm1TB9tqm9m6r4mttc1s3dfMmzWN/HbDXrp6+juNOVkRZkzMY+akPGb0vibmMrMkjxkT88jPTuXvIZHMoxvl5ITV1d3DzgOtbKttYuf+Vnbub+FP+1vYeSBYbkroeQCU5MepmJRHxcRcphfnMK04l+kTgvdpE3Iozc/W/Rwy7qTlPgiRVItFI8wuDc5RDOTuHGzpDAMjDI4wRNZXN/Db9Xtp7+o55JisqDG1NziKc5g2oT9IphTlMLkom5L8uOapknFDASEZycyYmB9nYn6cs2ZMOGy7u3OgpZPqg63srm9jd30r1QeD990H26jccYA9r+8+ZAgLIGIwKT+byYXZTC4K3wtz+pbLCnPC92xysqLHq7kiKaGAkHHJzJiUH2dSfpwF5cVJ9+npcWqb2qmub2NvQxs1je3sC9+DVxvrqxuobWqnJ8lIbXFuFpMLs5mUH6e0IHgvKYhTkh+npHc9XJ6Qm6XhLRlzFBAig4hEjMlFOUw+yuSE3T1OXXM7NQ3t7AuDo6ahP0T2N3ewYU8DdU0dffNZHfZZRl9gleRnM6kgTml+nEn52UzKz6I4L86E3Cwm5sWZkJdFcV4WhdkxXbUlKaWAEDlG0YgFw0yFR5/ltrO7hwMtHdQ1dbC/uYPapnb2Nwfrdc0d7G9up66pgw1hz6ShrWvQvxWNGBNyg7DoDY9gOQiRiXn9wRKsxynKyaIgJ6Z7SWRIFBAix1FWNDLkMAHo6OqhvrWTgy0dHGzt5GBLsFzf2smBlo5gvbWT+pbOvvtI6ls7D7uCa6CC7BiFOTGKcrKC99yspOuFOVkUhe/Fub3rWeRkRdR7GQcUECJjWDwWoSw86T0cnd09HGzppL41DJGWIFAa2rpobOukoTV8b+uksa2LmsY2tuzroqE1WB94cn6gWMQSQiRGfjxGQXaM/PBVkB0N3weUxWOHlBdkxxQ2Y5gCQiQDZUVHFiwQXOHV2tlNYxgm9X1hkjxcGtu6aGrvYk9DG83tXTS1d9Pc3kVrZ/eQPi8aMfLi0cPCJDFEcuNR8rJi5MWjwXI8Sm5W7/Kh5XlZwf7xmC5HPlYKCBE5hJmFX7qxY3p6YHeP09zRRXN7V19wNIVh0tzeRXNHwnJ7d99y73ttY0ew3NFFS0c3HQPuWzmaWMT6QyMeIzcremiQxIMgOaQ8KyjPiUfJiUXIyYqGr2A5NytKdricE4uSFbWM7v0oIEQkJaIRoyg8ZzEaurp7aO3sprWjm5bw1drZ1b/cV94VLPftG+zT1hlsb2zroqahnZbOrr5jWju7GcmkEhGjP0RikTBY+gOlL1xiUbLDgOnf1h802QkB1LstOxYlOxYhO3E5FjmuN2oqIETkhBCLRiiMRlLytEJ3p62zpy9M2ru6aevsoa0zCI/e5bbObtq6emjvXe7sCbeH+3R1h9uC/RvaOmntCNYT/+bRzvEcSWYLd8sAAAWWSURBVDRi5MQiZGf1h8bkwhx+esMFo/gvElBAiMi4ZxYMR+XGo5Qcffdj1tXdQ1tXGEADAqk3TNq7wvewPFjvL+tb7uohN0V37SsgRESOs1g0QkE0QsEYn2FYp/lFRCQpBYSIiCSlgBARkaQUECIikpQCQkREklJAiIhIUgoIERFJSgEhIiJJmY9kApIxysz2ATtGeHgpUDuK1TkRqM3jg9o8Poy0zSe5e1myDRkVEMfCzCrdfWG663E8qc3jg9o8PqSizRpiEhGRpBQQIiKSlAKi313prkAaqM3jg9o8Pox6m3UOQkREklIPQkREklJAiIhIUuM+IMzsCjPbaGabzeyWdNdntJjZUjOrMbO1CWWTzOwpM9sUvk9M2Pbl8N9go5n9eXpqfWzMbIaZ/c7MNpjZOjP7XFiese02sxwze9nMXgvb/LWwPGPb3MvMomb2qpn9KlzP6Dab2XYzW2Nmq82sMixLbZvdfdy+gCiwBZgDxIHXgHnprtcote0S4BxgbULZfwC3hMu3ALeGy/PCtmcDs8N/k2i62zCCNk8DzgmXC4E3w7ZlbLsBAwrC5Szgj8D5mdzmhLZ/AXgI+FW4ntFtBrYDpQPKUtrm8d6DOA/Y7O5b3b0DeARYlOY6jQp3fx7YP6B4EXBfuHwf8KGE8kfcvd3dtwGbCf5tTijuvtvdXwmXG4ENQDkZ3G4PNIWrWeHLyeA2A5hZBfA+4J6E4oxu8yBS2ubxHhDlwM6E9aqwLFNNcffdEHyZApPD8oz7dzCzWcDZBL+oM7rd4VDLaqAGeMrdM77NwHeB/wv0JJRlepsdeNLMVpnZkrAspW0e20/MTj1LUjYer/vNqH8HMysAfg583t0bzJI1L9g1SdkJ12537wbeYmYTgGVmtuAIu5/wbTaz9wM17r7KzC4dyiFJyk6oNocudPdqM5sMPGVmbxxh31Fp83jvQVQBMxLWK4DqNNXleNhrZtMAwveasDxj/h3MLIsgHH7i7r8IizO+3QDufhB4FriCzG7zhcAHzWw7wbDwO83sQTK7zbh7dfheAywjGDJKaZvHe0CsBOaa2WwziwOLgcfSXKdUegz4eLj8ceCXCeWLzSzbzGYDc4GX01C/Y2JBV+FHwAZ3/3bCpoxtt5mVhT0HzCwXuBx4gwxus7t/2d0r3H0Wwf/ZZ9z9r8ngNptZvpkV9i4D7wHWkuo2p/vMfLpfwHsJrnbZAvxjuusziu16GNgNdBL8mrgOKAGeBjaF75MS9v/H8N9gI3Bluus/wjZfRNCNfh1YHb7em8ntBs4EXg3bvBb4aliesW0e0P5L6b+KKWPbTHCl5Wvha13vd1Wq26ypNkREJKnxPsQkIiKDUECIiEhSCggREUlKASEiIkkpIEREJCkFhMgwmFl3OJtm72vUZgA2s1mJs++KpNt4n2pDZLha3f0t6a6EyPGgHoTIKAjn6r81fDbDy2Z2Slh+kpk9bWavh+8zw/IpZrYsfI7Da2b29vBPRc3s7vDZDk+Gd0eLpIUCQmR4cgcMMX00YVuDu58H/IBgtlHC5fvd/UzgJ8BtYfltwHPufhbBczvWheVzgdvdfT5wELgqxe0RGZTupBYZBjNrcveCJOXbgXe6+9ZwwsA97l5iZrXANHfvDMt3u3upme0DKty9PeFvzCKYrntuuP4lIMvdv5H6lokcTj0IkdHjgywPtk8y7QnL3eg8oaSRAkJk9Hw04f2lcPlFghlHAa4BXgiXnwY+DX0P/Ck6XpUUGSr9OhEZntzw6W29nnD33ktds83sjwQ/vD4Wln0WWGpmfw/sAz4Zln8OuMvMriPoKXyaYPZdkTFD5yBERkF4DmKhu9emuy4io0VDTCIikpR6ECIikpR6ECIikpQCQkREklJAiIhIUgoIERFJSgEhIiJJ/X+HQQYgjHxOAAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAdFklEQVR4nO3de5ScdZ3n8fcnnU7SucglxADp3NSwJCHAQhty5CJrZuSqyIwKOLtGjgq4IDqjLggcWYfRdUZcV444GXAY4IwSRUQRWSPLxQuiECQhaQIkhAhNCDTJcOuqpLo73/2jnm7KTnen011PVz1Vn9c5fVL11FNV31+dk/rU8/s9z++niMDMzOrXmEoXYGZmleUgMDOrcw4CM7M65yAwM6tzDgIzszrnIDAzq3MOAjOzOucgsJom6X5J/yFpfAqvLUkXS1onqUNSm6RbJS0q93uZpclBYDVL0hzgeCCA96fwFt8CPgNcDOwPHAL8BDhtb19I0tjylmY2dA4Cq2UfBX4P3AgsK31A0kxJP5bULmmbpG+XPPZJSeslvS7pcUlH9X1hSfOAC4FzIuLeiNgZEbmI+F5EfC3Z535Jnyh5zsck/bbkfki6UNIGYIOk5ZKu7vM+P5X0d8ntgyXdltT8jKSLy/AZmTkIrKZ9FPhe8neSpOkAkhqAO4E/AXOAGcCK5LEPAf8zee5bKB5JbOvntZcCbRHx0Ahr/ABwDLAA+D5wliQltewHvBdYIWkM8DNgTVLvUuCzkk4a4fubOQisNkk6DpgN/DAiHgGeBj6SPLwYOBj4QkR0RMSOiOj5pf4J4J8i4uEo2hgRf+rnLaYCL5Sh1P8VEdsjIg/8hmI31vHJYx8EHoyILcA7gWkR8fcRUYiITcD1wNllqMHqnIPAatUy4JcR8XJy//u82T00E/hTRHT187yZFENjT7YBB424Sniu50YUZ4BcAZyTbPoIxaMZKIbawZJe6fkDLgOml6EGq3MeoLKaI6kJ+DDQIGlrsnk8sK+kIyh++c6SNLafMHgOePsQ3uYe4FpJLRGxaoB9OoCJJfcP7GefvtP/3gL8UtLXKHYZnVlS1zMRMW8ItZntFR8RWC36ANBNsd/9yORvPsWul48CD1Hs1vmapEmSJkg6Nnnud4HPSzo6OT30HZJm932DiNgAfAe4RdKJksYlr3O2pEuT3VYDfyVpoqR3AB/fU+ER8SjQntSxMiJeSR56CHhN0iWSmiQ1SDpM0juH8wGZlXIQWC1aBvxbRDwbEVt7/oBvA38DCHgf8A7gWaANOAsgIm4FvkKxK+l1iqeD7j/A+1ycvOa1wCsUu5TOpDioC/BNoAC8CNzEm908e3IL8BdJDSR1dSc1Hwk8A7xMMSz2GeJrmg1IXpjGzKy++YjAzKzOOQjMzOqcg8DMrM45CMzM6lzmriM44IADYs6cOZUuw8wsUx555JGXI2Jaf49lLgjmzJnDqlUDXb9jZmb9kdTfVCmAu4bMzOqeg8DMrM45CMzM6pyDwMyszjkIzMzqXGpBIOkGSS9JWjfA45J0jaSNkh7rbzlAMzNLX5pHBDcCJw/y+CnAvOTvPOCfU6zFzMwGkNp1BBHxa0lzBtnlDODmZFWm30vaV9JBEVGO5f8yZWXrVlqff7XSZZhZlWuZsz8nHNLvNWEjUskLymZQskwfxTnhZ9DPOrCSzqN41MCsWbNGpbjRdNmP17Kto0BxyXIzs/5d8O6311wQ9Pe11+/iCBFxHXAdQEtLS80toPDGzi7OP+FtfPHU+ZUuxczqUCXPGmqjuFB4j2ZgS4VqqZjuXcHOrl00jWuodClmVqcqGQR3AB9Nzh5aArxaj+MD+c5uACY6CMysQlLrGpJ0C3AicICkNuBKoBEgIpYDdwGnAhuBHHBuWrVUs1yhC4CmcZmb/8/MakSaZw2ds4fHA7gwrffPinwhOSJo9BGBmVWGryyusFzBXUNmVlkOggrrCQIPFptZpTgIKqxnjGCixwjMrEIcBBXmriEzqzQHQYXl3TVkZhXmIKgwHxGYWaU5CCqsd4yg0WMEZlYZ/vYZZRHBytatvJrvBODhzdsBdw2ZWeU4CEbZxpfe4IJ//+OfbXvrlPGMG+uDMzOrDAfBKOs5EvjmWUdwzNypAOw7sbGSJZlZnXMQjLKeweHm/SZy8L5NFa7GzMyDxaOu90pizy1kZlXCQTDK8p09VxI7CMysOjgIRtmb1w24V87MqoODYJT1Tjs93kcEZlYdHASjLOf1B8ysyjgIRlmu0M24hjGMbfBHb2bVwd9Goyxf6PJVxGZWVRwEoyxX6PYZQ2ZWVRwEoyzX2e0jAjOrKg6CUZb3EYGZVRmfzD4KIoI1ba/yxo4utr66g8nj/bGbWfXwN9IoePyF1/jAtQ/03j910YEVrMbM7M85CEbBtjcKAFz1gcM49MApHDJ9SoUrMjN7k4NgFPRcRHbUrH1ZePA+Fa7GzOzPebB4FLw50Zxz18yqj4NgFHiBejOrZg6CUdAz0ZyvHzCzauQgGAWeaM7MqpmDYBR4ojkzq2b+ZhoFnmjOzKqZg2AUeKI5M6tmDoJR4InmzKyapRoEkk6W9KSkjZIu7efx/STdLukxSQ9JOizNeirFE82ZWTVLLQgkNQDXAqcAC4BzJC3os9tlwOqIOBz4KPCttOqppFyhi4mNvpjMzKpTmkcEi4GNEbEpIgrACuCMPvssAO4BiIgngDmSpqdYU0XkC91erN7MqlaaQTADeK7kfluyrdQa4K8AJC0GZgPNKdZUER3uGjKzKpZmEKifbdHn/teA/SStBj4NPAp07fZC0nmSVkla1d7eXv5KU5YvdNPkriEzq1Jpfju1ATNL7jcDW0p3iIjXgHMBJAl4Jvmjz37XAdcBtLS09A2TqpcrdPmIwMyqVppHBA8D8yTNlTQOOBu4o3QHSfsmjwF8Avh1Eg41xdcRmFk1S+2IICK6JF0ErAQagBsiolXSBcnjy4H5wM2SuoHHgY+nVU+ldO8Kdnbt8nUEZla1Uu24joi7gLv6bFtecvtBYF6aNVRavtNTUJtZdfOVxSnLFYpj301elMbMqpSDIGV5T0FtZlXOQZAyr05mZtXOQZCynFcnM7Mq5yBIWW/XkMcIzKxKOQhS1jNY7K4hM6tWDoKU9Zw+6q4hM6tWDoKUebDYzKqdgyBlHTuTriFPOmdmVcpBkLK8zxoysyrnIEhZrrObsWPEuLH+qM2sOvnbKWX5gheuN7Pq5iBImdciMLNq5yBIWXEtAg8Um1n1chCkLO9FacysyjkIUubVycys2jkIUpbr7PZaBGZW1RwEKcsXurwWgZlVNQdBytw1ZGbVzkGQMl9HYGbVzkGQMh8RmFm1cxCkaNeuIO/BYjOrcg6ClPxpWwfvvvo+ACb5iMDMqpiDICVPbn2d57bnef8RB3PqooMqXY6Z2YAcBCnpWZns4qXzmLn/xApXY2Y2MAdBSrwymZllhYMgJQ4CM8sKB0FK8oXiEpW+hsDMqp2DICW5QjcNY8S4Bn/EZlbd/C2Vkp4LySRVuhQzs0E5CFLidQjMLCscBCnJdXplMjPLBgdBSvKFLpo8/bSZZYCDICWebM7MsiLVIJB0sqQnJW2UdGk/j+8j6WeS1khqlXRumvWMppynnzazjEgtCCQ1ANcCpwALgHMkLeiz24XA4xFxBHAi8A1J49KqaTR5sNjMsiLNI4LFwMaI2BQRBWAFcEaffQKYouI5lpOB7UBXijWNmlxnlweLzSwT0gyCGcBzJffbkm2lvg3MB7YAa4HPRMSuvi8k6TxJqyStam9vT6vesvLKZGaWFWkGQX9XUkWf+ycBq4GDgSOBb0t6y25PirguIloiomXatGnlrzQFuUK3F603s0xIMwjagJkl95sp/vIvdS7w4yjaCDwDHJpiTaMiorgymccIzCwL0gyCh4F5kuYmA8BnA3f02edZYCmApOnAfwI2pVjTqNjRuYsIvESlmWVCat9UEdEl6SJgJdAA3BARrZIuSB5fDlwF3ChpLcWupEsi4uW0ahotuWTm0UnjfURgZtUv1Z+sEXEXcFefbctLbm8B3ptmDZXQsxaBryw2syzwlcUpeHNRGncNmVn1cxCkoKdryIPFZpYFewwCSZMkjSm5P0aSV2MfRL6na8hBYGYZMJQjgnuA0i/+icD/S6ec2uD1is0sS4YSBBMi4o2eO8ltHxEMItfpIDCz7BhKEHRIOqrnjqSjgXx6JWXfmwvXe7DYzKrfUL6pPgvcKqnnquCDgLPSKyn7eruGfPqomWXAHoMgIh6WdCjFq34FPBERnalXllG7dgVXr3wS8GCxmWXDUM4auhCYFBHrImItMFnSf0+/tGx6/pU8HYVu9mlqZPxYn51rZtVvKN9Un4yIV3ruRMR/AJ9Mr6RsyycDxV858zCKyyyYmVW3oQTBGJV8oyUrj9XEKmJp8KmjZpY1QxksXgn8UNJyiusJXAD831SryrCeq4qbGn3GkJllw1C+rS4BzgM+RXGw+FGKZw5ZP/I+IjCzjNlj11CydOTvKa4T0EJx/YD1KdeVWR1JEHgKajPLigGPCCQdQnExmXOAbcAPACLiv4xOadnki8nMLGsG+7Z6AvgN8L5kGUkk/e2oVJVhvpjMzLJmsK6hvwa2AvdJul7SUvpfkN5K5DzzqJllzIBBEBG3R8RZFBeTvx/4W2C6pH+WVHOripVLvtDNGOGLycwsM4YyWNwREd+LiNOBZmA1cGnqlWVUrtDNxHFjfTGZmWXGXv1sjYjtEfEvEfGetArKunxnl7uFzCxT3H9RZsUjAgeBmWWHz3Esg99saOdXT7YDsOa5V2jyGUNmliEOgjL45t1PsbokAM48akaFKzIzGzoHQRl07OzmLxdM51/+W0ulSzEz22seIyiDXGcXE30lsZlllIOgDPKFbp8pZGaZ5SAog1yh21NKmFlmOQhGKCLId/qUUTPLLgfBCO3o3EWEZxs1s+xyEIxQz4pkPiIws6xyEIyQZxs1s6xzEIxQvtNLU5pZtjkIRqjniGCSxwjMLKNSDQJJJ0t6UtJGSbtNXS3pC5JWJ3/rJHVL2j/Nmsot17s0pY8IzCybUgsCSQ3AtcApwALgHEkLSveJiK9HxJERcSTwReBXEbE9rZrSkNvpriEzy7Y0jwgWAxsjYlNEFIAVwBmD7H8OcEuK9aQi5zECM8u4NINgBvBcyf22ZNtuJE0ETgZuG+Dx8yStkrSqvb297IWORL63a8hjBGaWTWkGQX9rNcYA+74PeGCgbqGIuC4iWiKiZdq0aWUrsBx6Bos9xYSZZVWaQdAGzCy53wxsGWDfs8lgtxD4OgIzy740g+BhYJ6kuZLGUfyyv6PvTpL2Ad4N/DTFWlKTL3QzRjB+rM/ENbNsSq1jOyK6JF0ErAQagBsiolXSBcnjy5NdzwR+GREdadWSpuIaxWOR+usJMzOrfqmOcEbEXcBdfbYt73P/RuDGNOtIU76zy91CZpZp7s8YoeIRgYPAzLLLQTBCuUJ376L1ZmZZ5CAYobyPCMws4xwEI5QreOF6M8s2B8EI5bxwvZllnINghLxesZllnYNghDp2OgjMLNscBCOUL3TR1OgxAjPLLgfBCEQEOXcNmVnGOQhGYGfXLiI84ZyZZZuDYAR6p6B2EJhZhjkIRqBnvWIvXG9mWeYgGIG81yIwsxrgIBgBdw2ZWS1wEIyAVyczs1rgIBiBfGdxjMBzDZlZljkIRsBdQ2ZWCxwEI9DbNeT1CMwswxwEI5D3EYGZ1QAHwQi82TXkMQIzyy4HwQjkCl1IMKHRH6OZZZe/wUbgia2vM65hDJIqXYqZ2bA5CIZpU/sb3P34izSMcQiYWbY5CIbpxdd2AvDFUw6tcCVmZiPjIBimnovJDpuxT4UrMTMbGQfBMPmMITOrFQ6CYfJVxWZWKxwEw+QpqM2sVjgIhslHBGZWKxwEw5RPViebMNZBYGbZ5iAYplyhm6bGBsb4OgIzyzgHwTDlOrvdLWRmNcFBMEz5QjcTxzsIzCz7HATDlCt0MbHR1xCYWfalGgSSTpb0pKSNki4dYJ8TJa2W1CrpV2nWU065QrdPHTWzmpDaT1pJDcC1wF8CbcDDku6IiMdL9tkX+A5wckQ8K+mtadVTDq/mO9m1KwB4fUeXxwjMrCak2bexGNgYEZsAJK0AzgAeL9nnI8CPI+JZgIh4KcV6RuS2R9r43K1r/mzbSQunV6gaM7PySTMIZgDPldxvA47ps88hQKOk+4EpwLci4ua+LyTpPOA8gFmzZqVS7J5s3taBBFeevqB32/GHTKtILWZm5ZRmEPR3gn308/5HA0uBJuBBSb+PiKf+7EkR1wHXAbS0tPR9jVGRK3QzsbGBjx07txJvb2aWmjSDoA2YWXK/GdjSzz4vR0QH0CHp18ARwFNUmVyhiybPNGpmNSjNs4YeBuZJmitpHHA2cEeffX4KHC9prKSJFLuO1qdY07DlCr6AzMxqU2o/cSOiS9JFwEqgAbghIlolXZA8vjwi1kv6BfAYsAv4bkSsS6umkXAQmFmtSrWvIyLuAu7qs215n/tfB76eZh3lkPd1A2ZWo3xl8RDlCr5uwMxqk4NgiIqzjXqw2Mxqj4NgiPKebdTMapSDYIg8WGxmtcpBMEQeLDazWuVO7yGICA8Wm6Wos7OTtrY2duzYUelSMm/ChAk0NzfT2Ng45Oc4CIZgZ9cudgVM9JXFZqloa2tjypQpzJkzB8nLvw5XRLBt2zba2tqYO3fo0+H4m20QP139PJffvo7uZOppHxGYpWPHjh0OgTKQxNSpU2lvb9+r5zkIBrHmuVcpdO1i2btmM7ZhDKctOqjSJZnVLIdAeQznc3QQDCLf2cU+Exu5/LQFe97ZzCyjfNbQIHzKqJnVAwfBIIpXEzsIzKy2OQgGkfcRgVlduuiii5g9e3alyxg1DoJBFK8d8DCKWT155plnuP/++ykUCrz++uupvU93d3dqr723/C03iFyhm6mTx1e6DLO68uWftfL4ltfK+poLDn4LV75v4ZD2vfLKK7niiiu4/vrraW1tZcmSJQBs2bKFT3/602zatIl8Ps/NN99Mc3PzbtsWL17MkiVLWLFiBXPmzOH555/njDPOYNWqVXzoQx9i5syZPProoyxdupRDDz2Uq6++mnw+z5QpU7j99tuZNm1av+/V1NTEBRdcwAMPPADAH//4Rz7/+c9z7733jvjzcRAMwhPNmdWX1tZW1q1bx0033cRvf/vb3iDo6urilFNO4Stf+Qqnn346uVyO7u5ujjvuuN22RQTPPvtsb9fSY489xqJFiwBYu3Yt8+fP57777gNg27ZtfPCDHwTgy1/+Mj/84Q85//zz+32vSZMm8fTTT9Pd3U1DQwOf+9zn+MY3vlGWdjsIBuGzhsxG31B/uafh8ssv56qrrkIS8+fPZ9264oKJP/nJT5g/fz6nn346ABMnTuRHP/rRbtsANmzYwNy5c3vP5+8Jgh07drB9+3a+9KUv9b7fjTfeyA9+8AN27tzJ1q1b+epXv9rve/VYuHAhra2tbNiwgVmzZnHUUUeVpd0OgkHkvQaBWd34wx/+wMqVK1m9ejUXXnghO3bs4PDDDwdg9erVvV1EPfrbBsVf/T1HAACrVq3i/PPPp7W1lWOOOYaxY4vfKTfffDMPPfQQ9957L5MnT+aEE05g4cKF3Hnnnf2+LsCSJUt44IEH+M53vsMvfvGLcjXdg8UDiQg6PNGcWd247LLLuPPOO9m8eTObN29mzZo1vUcEBx54IK2trb37tre397sNYPv27TQ1NQGwfv16fv7zn7No0SLWrl3bGyxQDIx3vetdTJ48mdtuu43f/e53LFq0aMDXhWIQXHHFFZx55pnMmDGjbG13EAxgZ9cuIvDU02Z14O6772bnzp0sXbq0d9v06dPp6Ohg+/btfOxjH+PFF19k4cKFHHnkkTz44IP9bgM46aSTuOeee/jwhz/MrbfeytSpU5k+ffpuQbBs2TKuueYajj/+eJ566ine9ra3MWnSpAFfF+DQQw9l/PjxXHLJJWVtvyKirC+YtpaWlli1alXq77O9o8BRV93Nle9bwLnHDn0WPzPbe+vXr2f+/PmVLqPqXXTRRbzzne9k2bJlg+7X3+cp6ZGIaOlv/7rpAP/VU+38w52PD3n/Ls84amZV4umnn+a0007j2GOP3WMIDEfdBMHk8WOZN33yXj3niOZ9OG7etJQqMjMbmre//e088cQTqb1+3QTB0bP34+jZR1e6DDOzquPBYjOzOucgMLOqkLUTV6rVcD5HB4GZVdyECRPYtm2bw2CEetYsnjBhwl49r27GCMysejU3N9PW1rbXa+3a7iZMmEBzc/NePcdBYGYV19jYyNy5vl6nUtw1ZGZW5xwEZmZ1zkFgZlbnMjfXkKR24E/DfPoBwMtlLCcL3Ob64DbXh5G0eXZE9DtVQuaCYCQkrRpo0qVa5TbXB7e5PqTVZncNmZnVOQeBmVmdq7cguK7SBVSA21wf3Ob6kEqb62qMwMzMdldvRwRmZtaHg8DMrM7VTRBIOlnSk5I2Srq00vWUi6QbJL0kaV3Jtv0l3S1pQ/LvfiWPfTH5DJ6UdFJlqh4ZSTMl3SdpvaRWSZ9JttdsuyVNkPSQpDVJm7+cbK/ZNgNIapD0qKQ7k/s13V4ASZslrZW0WtKqZFu67Y6Imv8DGoCngbcB44A1wIJK11Wmtp0AHAWsK9n2T8Clye1LgX9Mbi9I2j4emJt8Jg2VbsMw2nwQcFRyewrwVNK2mm03IGBycrsR+AOwpJbbnLTj74DvA3cm92u6vUlbNgMH9NmWarvr5YhgMbAxIjZFRAFYAZxR4ZrKIiJ+DWzvs/kM4Kbk9k3AB0q2r4iInRHxDLCR4meTKRHxQkT8Mbn9OrAemEENtzuK3kjuNiZ/QQ23WVIzcBrw3ZLNNdvePUi13fUSBDOA50rutyXbatX0iHgBil+awFuT7TX3OUiaA/xnir+Qa7rdSTfJauAl4O6IqPU2/x/gfwC7SrbVcnt7BPBLSY9IOi/Zlmq762U9AvWzrR7Pm62pz0HSZOA24LMR8ZrUX/OKu/azLXPtjohu4EhJ+wK3SzpskN0z3WZJpwMvRcQjkk4cylP62ZaZ9vZxbERskfRW4G5JTwyyb1naXS9HBG3AzJL7zcCWCtUyGl6UdBBA8u9Lyfaa+RwkNVIMge9FxI+TzTXfboCIeAW4HziZ2m3zscD7JW2m2JX7Hkn/Tu22t1dEbEn+fQm4nWJXT6rtrpcgeBiYJ2mupHHA2cAdFa4pTXcAy5Lby4Cflmw/W9J4SXOBecBDFahvRFT86f+vwPqI+N8lD9VsuyVNS44EkNQE/AXwBDXa5oj4YkQ0R8Qciv9f742I/0qNtreHpEmSpvTcBt4LrCPtdld6hHwUR+JPpXh2ydPA5ZWup4ztugV4Aeik+Ovg48BU4B5gQ/Lv/iX7X558Bk8Cp1S6/mG2+TiKh7+PAauTv1Nrud3A4cCjSZvXAV9Kttdsm0vacSJvnjVU0+2leGbjmuSvtee7Ku12e4oJM7M6Vy9dQ2ZmNgAHgZlZnXMQmJnVOQeBmVmdcxCYmdU5B4FZH5K6k5kfe/7KNlutpDmlM8WaVYN6mWLCbG/kI+LIShdhNlp8RGA2RMk88f+YrAvwkKR3JNtnS7pH0mPJv7OS7dMl3Z6sIbBG0ruSl2qQdH2yrsAvkyuFzSrGQWC2u6Y+XUNnlTz2WkQsBr5NcXZMkts3R8ThwPeAa5Lt1wC/iogjKK4Z0ZpsnwdcGxELgVeAv065PWaD8pXFZn1IeiMiJvezfTPwnojYlEx6tzUipkp6GTgoIjqT7S9ExAGS2oHmiNhZ8hpzKE4hPS+5fwnQGBH/kH7LzPrnIwKzvRMD3B5on/7sLLndjcfqrMIcBGZ756ySfx9Mbv+O4gyZAH8D/Da5fQ/wKehdVOYto1Wk2d7wLxGz3TUlK4H1+EVE9JxCOl7SHyj+iDon2XYxcIOkLwDtwLnJ9s8A10n6OMVf/p+iOFOsWVXxGIHZECVjBC0R8XKlazErJ3cNmZnVOR8RmJnVOR8RmJnVOQeBmVmdcxCYmdU5B4GZWZ1zEJiZ1bn/D3LAMOtGaamYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 利用鸢尾花数据集，实现前向传播、反向传播，可视化loss曲线\n",
    "\n",
    "# 导入所需模块\n",
    "import tensorflow as tf\n",
    "from sklearn import datasets\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import time  ##1##\n",
    "\n",
    "# 导入数据，分别为输入特征和标签\n",
    "x_data = datasets.load_iris().data\n",
    "y_data = datasets.load_iris().target\n",
    "\n",
    "# 随机打乱数据（因为原始数据是顺序的，顺序不打乱会影响准确率）\n",
    "# seed: 随机数种子，是一个整数，当设置之后，每次生成的随机数都一样（为方便教学，以保每位同学结果一致）\n",
    "np.random.seed(116)  # 使用相同的seed，保证输入特征和标签一一对应\n",
    "np.random.shuffle(x_data)\n",
    "np.random.seed(116)\n",
    "np.random.shuffle(y_data)\n",
    "tf.random.set_seed(116)\n",
    "\n",
    "# 将打乱后的数据集分割为训练集和测试集，训练集为前120行，测试集为后30行\n",
    "x_train = x_data[:-30]\n",
    "y_train = y_data[:-30]\n",
    "x_test = x_data[-30:]\n",
    "y_test = y_data[-30:]\n",
    "\n",
    "# 转换x的数据类型，否则后面矩阵相乘时会因数据类型不一致报错\n",
    "x_train = tf.cast(x_train, tf.float32)\n",
    "x_test = tf.cast(x_test, tf.float32)\n",
    "\n",
    "# from_tensor_slices函数使输入特征和标签值一一对应。（把数据集分批次，每个批次batch组数据）\n",
    "train_db = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(32)\n",
    "test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)\n",
    "\n",
    "# 生成神经网络的参数，4个输入特征故，输入层为4个输入节点；因为3分类，故输出层为3个神经元\n",
    "# 用tf.Variable()标记参数可训练\n",
    "# 使用seed使每次生成的随机数相同（方便教学，使大家结果都一致，在现实使用时不写seed）\n",
    "w1 = tf.Variable(tf.random.truncated_normal([4, 3], stddev=0.1, seed=1))\n",
    "b1 = tf.Variable(tf.random.truncated_normal([3], stddev=0.1, seed=1))\n",
    "\n",
    "lr = 0.1  # 学习率为0.1\n",
    "train_loss_results = []  # 将每轮的loss记录在此列表中，为后续画loss曲线提供数据\n",
    "test_acc = []  # 将每轮的acc记录在此列表中，为后续画acc曲线提供数据\n",
    "epoch = 500  # 循环500轮\n",
    "loss_all = 0  # 每轮分4个step，loss_all记录四个step生成的4个loss的和\n",
    "\n",
    "##########################################################################\n",
    "v_w, v_b = 0, 0\n",
    "##########################################################################\n",
    "\n",
    "# 训练部分\n",
    "now_time = time.time()  ##2##\n",
    "for epoch in range(epoch):  # 数据集级别的循环，每个epoch循环一次数据集\n",
    "    for step, (x_train, y_train) in enumerate(train_db):  # batch级别的循环 ，每个step循环一个batch\n",
    "        with tf.GradientTape() as tape:  # with结构记录梯度信息\n",
    "            y = tf.matmul(x_train, w1) + b1  # 神经网络乘加运算\n",
    "            y = tf.nn.softmax(y)  # 使输出y符合概率分布（此操作后与独热码同量级，可相减求loss）\n",
    "            y_ = tf.one_hot(y_train, depth=3)  # 将标签值转换为独热码格式，方便计算loss和accuracy\n",
    "            loss = tf.reduce_mean(tf.square(y_ - y))  # 采用均方误差损失函数mse = mean(sum(y-out)^2)\n",
    "            loss_all += loss.numpy()  # 将每个step计算出的loss累加，为后续求loss平均值提供数据，这样计算的loss更准确\n",
    "        # 计算loss对各个参数的梯度\n",
    "        grads = tape.gradient(loss, [w1, b1])\n",
    "\n",
    "        ##########################################################################\n",
    "        # adagrad\n",
    "        v_w += tf.square(grads[0])\n",
    "        v_b += tf.square(grads[1])\n",
    "        w1.assign_sub(lr * grads[0] / tf.sqrt(v_w))\n",
    "        b1.assign_sub(lr * grads[1] / tf.sqrt(v_b))\n",
    "    ##########################################################################\n",
    "\n",
    "    # 每个epoch，打印loss信息\n",
    "    print(\"Epoch {}, loss: {}\".format(epoch, loss_all / 4))\n",
    "    train_loss_results.append(loss_all / 4)  # 将4个step的loss求平均记录在此变量中\n",
    "    loss_all = 0  # loss_all归零，为记录下一个epoch的loss做准备\n",
    "\n",
    "    # 测试部分\n",
    "    # total_correct为预测对的样本个数, total_number为测试的总样本数，将这两个变量都初始化为0\n",
    "    total_correct, total_number = 0, 0\n",
    "    for x_test, y_test in test_db:\n",
    "        # 使用更新后的参数进行预测\n",
    "        y = tf.matmul(x_test, w1) + b1\n",
    "        y = tf.nn.softmax(y)\n",
    "        pred = tf.argmax(y, axis=1)  # 返回y中最大值的索引，即预测的分类\n",
    "        # 将pred转换为y_test的数据类型\n",
    "        pred = tf.cast(pred, dtype=y_test.dtype)\n",
    "        # 若分类正确，则correct=1，否则为0，将bool型的结果转换为int型\n",
    "        correct = tf.cast(tf.equal(pred, y_test), dtype=tf.int32)\n",
    "        # 将每个batch的correct数加起来\n",
    "        correct = tf.reduce_sum(correct)\n",
    "        # 将所有batch中的correct数加起来\n",
    "        total_correct += int(correct)\n",
    "        # total_number为测试的总样本数，也就是x_test的行数，shape[0]返回变量的行数\n",
    "        total_number += x_test.shape[0]\n",
    "    # 总的准确率等于total_correct/total_number\n",
    "    acc = total_correct / total_number\n",
    "    test_acc.append(acc)\n",
    "    print(\"Test_acc:\", acc)\n",
    "    print(\"--------------------------\")\n",
    "total_time = time.time() - now_time  ##3##\n",
    "print(\"total_time\", total_time)  ##4##\n",
    "\n",
    "# 绘制 loss 曲线\n",
    "plt.title('Loss Function Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Loss')  # y轴变量名称\n",
    "plt.plot(train_loss_results, label=\"$Loss$\")  # 逐点画出trian_loss_results值并连线，连线图标是Loss\n",
    "plt.legend()  # 画出曲线图标\n",
    "plt.show()  # 画出图像\n",
    "\n",
    "# 绘制 Accuracy 曲线\n",
    "plt.title('Acc Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Acc')  # y轴变量名称\n",
    "plt.plot(test_acc, label=\"$Accuracy$\")  # 逐点画出test_acc值并连线，连线图标是Accuracy\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# 请将loss曲线、ACC曲线、total_time记录到 class2\\优化器对比.docx  对比各优化器收敛情况\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RMSProp算法-在SGD基础增加了二阶动量\n",
    "相比于AdaGrad的历史梯度,RMSProp增加了一个衰减系数来控制历史信息的获取多少\n",
    "$$ m_t = g_t $$\n",
    "$$ V_t = \\beta*v_{t-1}+(1-\\beta)g_t^2 $$\n",
    "$$ \\eta _t =\\frac{lr*m_t}{\\sqrt{V_t}} = \\frac{lr*g_t}{\\sqrt{\\beta*v_{t-1}+(1-\\beta)g_t^2}} $$\n",
    "$$ w_{t+1} = w_t - \\eta _t = w_t - \\frac{lr*g_t}{\\sqrt{\\beta*v_{t-1}+(1-\\beta)g_t^2}} $$\n",
    "\n",
    "        v_w, v_b = 0, 0  \n",
    "        beta = 0.9  \n",
    "        v_w = beta * v_w + (1 - beta) * tf.square(grads[0])  \n",
    "        v_b = beta * v_b + (1 - beta) * tf.square(grads[1])  \n",
    "        w1.assign_sub(lr * grads[0] / tf.sqrt(v_w))  \n",
    "        b1.assign_sub(lr * grads[1] / tf.sqrt(v_b))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, loss: 0.3926013857126236\n",
      "Test_acc: 0.16666666666666666\n",
      "--------------------------\n",
      "Epoch 1, loss: 0.20467261411249638\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 2, loss: 0.17201940715312958\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 3, loss: 0.15918194875121117\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 4, loss: 0.15706615522503853\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 5, loss: 0.21816572919487953\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 6, loss: 0.14377840794622898\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 7, loss: 0.17571274936199188\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 8, loss: 0.18757154792547226\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 9, loss: 0.19608472101390362\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 10, loss: 0.09835131652653217\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 11, loss: 0.12403555028140545\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 12, loss: 0.18723830953240395\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 13, loss: 0.1682581827044487\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 14, loss: 0.10289529897272587\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 15, loss: 0.1912501584738493\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 16, loss: 0.10690359678119421\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 17, loss: 0.16707619652152061\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 18, loss: 0.09712469018995762\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 19, loss: 0.14692728035151958\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 20, loss: 0.1130000576376915\n",
      "Test_acc: 0.8333333333333334\n",
      "--------------------------\n",
      "Epoch 21, loss: 0.1312888916581869\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 22, loss: 0.14335670322179794\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 23, loss: 0.14184509962797165\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 24, loss: 0.09614701848477125\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 25, loss: 0.06645769346505404\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 26, loss: 0.09875406604260206\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 27, loss: 0.1214442141354084\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 28, loss: 0.13192315213382244\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 29, loss: 0.11436496302485466\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 30, loss: 0.09751834813505411\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 31, loss: 0.08348444756120443\n",
      "Test_acc: 0.8666666666666667\n",
      "--------------------------\n",
      "Epoch 32, loss: 0.03387557249516249\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 33, loss: 0.11853594146668911\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 34, loss: 0.09329163283109665\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 35, loss: 0.1171216955408454\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 36, loss: 0.10019966308027506\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 37, loss: 0.07477776752784848\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 38, loss: 0.0742293824441731\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 39, loss: 0.09871879126876593\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 40, loss: 0.070330910384655\n",
      "Test_acc: 0.9666666666666667\n",
      "--------------------------\n",
      "Epoch 41, loss: 0.07131194975227118\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 42, loss: 0.08518254524096847\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 43, loss: 0.0841852966696024\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 44, loss: 0.07050436967983842\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 45, loss: 0.07280493341386318\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 46, loss: 0.0577983814291656\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 47, loss: 0.0865863123908639\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 48, loss: 0.062310051172971725\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 49, loss: 0.06802953267470002\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 50, loss: 0.05585814639925957\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 51, loss: 0.07325917715206742\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 52, loss: 0.052181640872731805\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 53, loss: 0.08286307565867901\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 54, loss: 0.05124968430027366\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 55, loss: 0.07729693595319986\n",
      "Test_acc: 0.5666666666666667\n",
      "--------------------------\n",
      "Epoch 56, loss: 0.05052058631554246\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 57, loss: 0.07210225146263838\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 58, loss: 0.048650020034983754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 59, loss: 0.07164605148136616\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 60, loss: 0.047409142134711146\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 61, loss: 0.07039465848356485\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 62, loss: 0.04643832193687558\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 63, loss: 0.0687064602971077\n",
      "Test_acc: 0.6\n",
      "--------------------------\n",
      "Epoch 64, loss: 0.04564674268476665\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 65, loss: 0.06680149026215076\n",
      "Test_acc: 0.6333333333333333\n",
      "--------------------------\n",
      "Epoch 66, loss: 0.04502795566804707\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 67, loss: 0.06487354729324579\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 68, loss: 0.044569259975105524\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 69, loss: 0.06296336371451616\n",
      "Test_acc: 0.7\n",
      "--------------------------\n",
      "Epoch 70, loss: 0.044273311737924814\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 71, loss: 0.06105491053313017\n",
      "Test_acc: 0.7333333333333333\n",
      "--------------------------\n",
      "Epoch 72, loss: 0.04415719164535403\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 73, loss: 0.05909866653382778\n",
      "Test_acc: 0.7666666666666667\n",
      "--------------------------\n",
      "Epoch 74, loss: 0.04425172647461295\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 75, loss: 0.05703649390488863\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 76, loss: 0.0445945761166513\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 77, loss: 0.05482097342610359\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 78, loss: 0.045206377282738686\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 79, loss: 0.05245484132319689\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 80, loss: 0.046009110286831856\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 81, loss: 0.05007370561361313\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 82, loss: 0.04664710257202387\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 83, loss: 0.04797810036689043\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 84, loss: 0.046533601358532906\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 85, loss: 0.0463609267026186\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 86, loss: 0.04563643550500274\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 87, loss: 0.045101705472916365\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 88, loss: 0.04456120915710926\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 89, loss: 0.04407221032306552\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 90, loss: 0.04362203786149621\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 91, loss: 0.043201979249715805\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 92, loss: 0.04280423652380705\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 93, loss: 0.04242284270003438\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 94, loss: 0.04205410182476044\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 95, loss: 0.041695842519402504\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 96, loss: 0.04134663799777627\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 97, loss: 0.04100561048835516\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 98, loss: 0.040672131814062595\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 99, loss: 0.040345665998756886\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 100, loss: 0.04002587357535958\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 101, loss: 0.039712431374937296\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 102, loss: 0.03940500458702445\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 103, loss: 0.03910341626033187\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 104, loss: 0.03880737628787756\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 105, loss: 0.03851672587916255\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 106, loss: 0.03823128994554281\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 107, loss: 0.037950919941067696\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 108, loss: 0.0376754030585289\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 109, loss: 0.03740471042692661\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 110, loss: 0.03713862085714936\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 111, loss: 0.0368770444765687\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 112, loss: 0.03661994403228164\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 113, loss: 0.03636711137369275\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 114, loss: 0.03611843241378665\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 115, loss: 0.03587393742054701\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 116, loss: 0.03563341964036226\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 117, loss: 0.03539685998111963\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 118, loss: 0.03516412526369095\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 119, loss: 0.034935192903503776\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 120, loss: 0.034709946019575\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 121, loss: 0.034488326171413064\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 122, loss: 0.03427030146121979\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 123, loss: 0.034055703319609165\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 124, loss: 0.03384458855725825\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 125, loss: 0.03363689128309488\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 126, loss: 0.03343246388249099\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 127, loss: 0.03323126258328557\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 128, loss: 0.03303337167017162\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 129, loss: 0.03283857344649732\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 130, loss: 0.03264688909985125\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 131, loss: 0.032458312110975385\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 132, loss: 0.03227274143137038\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 133, loss: 0.03209020453505218\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 134, loss: 0.0319105323869735\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 135, loss: 0.03173386096023023\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 136, loss: 0.03156005195342004\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 137, loss: 0.03138910164125264\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 138, loss: 0.031220961827784777\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 139, loss: 0.031055682571604848\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 140, loss: 0.03089311276562512\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 141, loss: 0.030733278254047036\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 142, loss: 0.030576285207644105\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 143, loss: 0.03042199439369142\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 144, loss: 0.03027038392610848\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 145, loss: 0.030121515737846494\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 146, loss: 0.029975346056744456\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 147, loss: 0.029831886058673263\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 148, loss: 0.029691173927858472\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 149, loss: 0.029553109547123313\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 150, loss: 0.029417807003483176\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 151, loss: 0.029285250464454293\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 152, loss: 0.02915552817285061\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 153, loss: 0.029028499266132712\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 154, loss: 0.028904365142807364\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 155, loss: 0.02878308924846351\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 156, loss: 0.02866475679911673\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 157, loss: 0.028549401788040996\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 158, loss: 0.028437082888558507\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 159, loss: 0.02832793234847486\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 160, loss: 0.0282219504006207\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 161, loss: 0.02811931213364005\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 162, loss: 0.028020158875733614\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 163, loss: 0.027924553956836462\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 164, loss: 0.02783266664482653\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 165, loss: 0.02774469694122672\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 166, loss: 0.027660861844196916\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 167, loss: 0.02758133294992149\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 168, loss: 0.0275064236484468\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 169, loss: 0.027436402393504977\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 170, loss: 0.027371617266908288\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 171, loss: 0.027312430320307612\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 172, loss: 0.02725925575941801\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 173, loss: 0.027212625602260232\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 174, loss: 0.027173121343366802\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 175, loss: 0.0271412655711174\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 176, loss: 0.027117806603200734\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 177, loss: 0.027103488333523273\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 178, loss: 0.027099203667603433\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 179, loss: 0.027105812449008226\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 180, loss: 0.027124281390570104\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 181, loss: 0.027155703399330378\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 182, loss: 0.027200837736018002\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 183, loss: 0.027260823524557054\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 184, loss: 0.027336190338246524\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 185, loss: 0.027427394525147974\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 186, loss: 0.02753435412887484\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 187, loss: 0.027656389051117003\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 188, loss: 0.027791825705207884\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 189, loss: 0.027938233339227736\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 190, loss: 0.028092029970139265\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 191, loss: 0.028248872025869787\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 192, loss: 0.028403849923051894\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 193, loss: 0.02855214721057564\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 194, loss: 0.028689416765701026\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 195, loss: 0.028812545351684093\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 196, loss: 0.028919866948854178\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 197, loss: 0.029011088889092207\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 198, loss: 0.02908713137730956\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 199, loss: 0.029149639594834298\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 200, loss: 0.029200659890193492\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 201, loss: 0.02924223180161789\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 202, loss: 0.02927614323562011\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 203, loss: 0.029303984891157597\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 204, loss: 0.0293268752284348\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 205, loss: 0.02934590744553134\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 206, loss: 0.029361721768509597\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 207, loss: 0.02937487995950505\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 208, loss: 0.029385761416051537\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 209, loss: 0.029394637560471892\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 210, loss: 0.029401865496765822\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 211, loss: 0.029407549591269344\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 212, loss: 0.02941185235977173\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 213, loss: 0.029414930613711476\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 214, loss: 0.029416837787721306\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 215, loss: 0.029417784593533725\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 216, loss: 0.02941775316139683\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 217, loss: 0.02941691578598693\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 218, loss: 0.02941525896312669\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 219, loss: 0.029412837291602045\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 220, loss: 0.029409811017103493\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 221, loss: 0.02940609137294814\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 222, loss: 0.029401879117358476\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 223, loss: 0.029397130478173494\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 224, loss: 0.029391881253104657\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 225, loss: 0.029386204958427697\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 226, loss: 0.029380149149801582\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 227, loss: 0.029373676574323326\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 228, loss: 0.029366888978984207\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 229, loss: 0.029359779611695558\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 230, loss: 0.029352375713642687\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 231, loss: 0.029344671173021197\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 232, loss: 0.029336780513403937\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 233, loss: 0.029328645701752976\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 234, loss: 0.02932030489319004\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 235, loss: 0.029311806400073692\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 236, loss: 0.029303104005521163\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 237, loss: 0.02929426505579613\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 238, loss: 0.02928527895710431\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 239, loss: 0.029276190005475655\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 240, loss: 0.029266988043673337\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 241, loss: 0.02925770473666489\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 242, loss: 0.02924832809367217\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 243, loss: 0.02923887362703681\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 244, loss: 0.02922935938113369\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 245, loss: 0.029219785152236\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 246, loss: 0.029210170498117805\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 247, loss: 0.02920052778790705\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 248, loss: 0.029190867120632902\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 249, loss: 0.029181130463257432\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 250, loss: 0.029171432281145826\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 251, loss: 0.029161736834794283\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 252, loss: 0.029152062168577686\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 253, loss: 0.02914233287447132\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 254, loss: 0.029132641473552212\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 255, loss: 0.029122935520717874\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 256, loss: 0.029113335855072364\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 257, loss: 0.029103691136697307\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 258, loss: 0.029094099765643477\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 259, loss: 0.029084552021231502\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 260, loss: 0.029075043217744678\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 261, loss: 0.029065561189781874\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 262, loss: 0.02905615995405242\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 263, loss: 0.02904674253659323\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 264, loss: 0.029037429892923683\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 265, loss: 0.029028181277681142\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 266, loss: 0.029018974339123815\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 267, loss: 0.029009802849031985\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 268, loss: 0.02900068840244785\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 269, loss: 0.02899166999850422\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 270, loss: 0.02898272214224562\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 271, loss: 0.028973847161978483\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 272, loss: 0.028965031553525478\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 273, loss: 0.028956251044292003\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 274, loss: 0.02894757577450946\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 275, loss: 0.02893896837485954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 276, loss: 0.0289304296602495\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 277, loss: 0.02892198268091306\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 278, loss: 0.028913601126987487\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 279, loss: 0.02890529832802713\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 280, loss: 0.02889705024426803\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 281, loss: 0.02888891228940338\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 282, loss: 0.02888082090066746\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 283, loss: 0.028872817696537822\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 284, loss: 0.028864924446679652\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 285, loss: 0.02885705087101087\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 286, loss: 0.028849281079601496\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 287, loss: 0.02884161623660475\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 288, loss: 0.02883400209248066\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 289, loss: 0.028826480673160404\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 290, loss: 0.02881904598325491\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 291, loss: 0.028811653202865273\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 292, loss: 0.028804384171962738\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 293, loss: 0.02879714622395113\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 294, loss: 0.0287900252151303\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 295, loss: 0.028782969515305012\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 296, loss: 0.028775974235031754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 297, loss: 0.02876906911842525\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 298, loss: 0.028762233210727572\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 299, loss: 0.028755491075571626\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 300, loss: 0.028748789511155337\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 301, loss: 0.028742158494424075\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 302, loss: 0.028735648840665817\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 303, loss: 0.02872915583429858\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 304, loss: 0.028722762188408524\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 305, loss: 0.028716432163491845\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 306, loss: 0.028710175130981952\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 307, loss: 0.028703995514661074\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 308, loss: 0.02869788761017844\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 309, loss: 0.028691828716546297\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 310, loss: 0.02868586528347805\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 311, loss: 0.02867992129176855\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 312, loss: 0.028674118395429105\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 313, loss: 0.028668277722317725\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 314, loss: 0.028662580822128803\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 315, loss: 0.028656928800046444\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 316, loss: 0.028651322470977902\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 317, loss: 0.02864578430308029\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 318, loss: 0.028640304051805288\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 319, loss: 0.02863487444119528\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 320, loss: 0.02862953330622986\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 321, loss: 0.028624211146961898\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 322, loss: 0.028618984448257834\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 323, loss: 0.028613795817364007\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 324, loss: 0.028608649969100952\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 325, loss: 0.02860358712496236\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 326, loss: 0.028598553559277207\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 327, loss: 0.028593569004442543\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 328, loss: 0.028588670305907726\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 329, loss: 0.028583801875356585\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 330, loss: 0.028579018835444003\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 331, loss: 0.02857422735542059\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 332, loss: 0.028569548157975078\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 333, loss: 0.028564865642692894\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 334, loss: 0.02856025326764211\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 335, loss: 0.028555696830153465\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 336, loss: 0.028551162220537663\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 337, loss: 0.02854671236127615\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 338, loss: 0.02854229713557288\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 339, loss: 0.028537941456306726\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 340, loss: 0.028533629432786256\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 341, loss: 0.02852931182133034\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 342, loss: 0.028525110974442214\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 343, loss: 0.028520898893475533\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 344, loss: 0.028516779013443738\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 345, loss: 0.02851268951781094\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 346, loss: 0.028508625458925962\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 347, loss: 0.02850462030619383\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 348, loss: 0.028500668238848448\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 349, loss: 0.028496715996880084\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 350, loss: 0.028492851357441396\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 351, loss: 0.028489045798778534\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 352, loss: 0.028485241520684212\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 353, loss: 0.028481501329224557\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 354, loss: 0.0284778272616677\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 355, loss: 0.02847417100565508\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 356, loss: 0.028470550198107958\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 357, loss: 0.028467001859098673\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 358, loss: 0.02846350334584713\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 359, loss: 0.028460022702347487\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 360, loss: 0.02845661184983328\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 361, loss: 0.028453199949581176\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 362, loss: 0.02844989293953404\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 363, loss: 0.02844661148265004\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 364, loss: 0.02844334440305829\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 365, loss: 0.028440149850212038\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 366, loss: 0.028437017695978284\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 367, loss: 0.02843393210787326\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 368, loss: 0.02843086840584874\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 369, loss: 0.028427927754819393\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 370, loss: 0.028424928663298488\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 371, loss: 0.028422031202353537\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 372, loss: 0.028419220936484635\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 373, loss: 0.02841638762038201\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 374, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 375, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 376, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 377, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 378, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 379, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 380, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 381, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 382, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 383, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 384, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 385, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 386, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 387, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 388, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 389, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 390, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 391, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 392, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 393, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 394, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 395, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 396, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 397, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 398, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 399, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 400, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 401, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 402, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 403, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 404, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 405, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 406, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 407, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 408, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 409, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 410, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 411, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 412, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 413, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 414, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 415, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 416, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 417, loss: nan\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 418, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 419, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 420, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 421, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 422, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 423, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 424, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 425, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 426, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 427, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 428, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 429, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 430, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 431, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 432, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 433, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 434, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 435, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 436, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 437, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 438, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 439, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 440, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 441, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 442, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 443, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 444, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 445, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 446, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 447, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 448, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 449, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 450, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 451, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 452, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 453, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 454, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 455, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 456, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 457, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 458, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 459, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 460, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 461, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 462, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 463, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 464, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 465, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 466, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 467, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 468, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 469, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 470, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 471, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 472, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 473, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 474, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 475, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 476, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 477, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 478, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 479, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 480, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 481, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 482, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 483, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 484, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 485, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 486, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 487, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 488, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 489, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 490, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 491, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 492, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 493, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 494, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 495, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 496, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 497, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 498, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "Epoch 499, loss: nan\n",
      "Test_acc: 0.36666666666666664\n",
      "--------------------------\n",
      "total_time 11.33805227279663\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xcdX3/8ddnZnd2s5fcdjcQciEBohiUmzFqiSjeSqQ2WH0g6k9bC7/8sNDWiv6krfWn1baitbUoiqgURJF6A6NGwKKIFFSWNuRGAiEEsiQhu7ntJXub3c/vj3Nm98zO2Wv27Ew27+fDeeyZ7zln5jMHM5/5fr/n+/2auyMiIjJUqtgBiIhIaVKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCFSBGbWbmanFTsOkZEoQUjRmNkuM3tjEd73VjPrCb+kc493Jvh+D5jZldEyd69x950Jvd+7zawx/Fx7zexnZrYqifeS6U0JQk5Unw2/pHOP/yh2QJPBzD4EfAH4R+AkYDHwZWDNBF6rbHKjk+ONEoSUHDOrMLMvmNme8PEFM6sI99Wb2U/M7LCZHTSzX5tZKtz3UTN73szazGy7mb1hnO97q5l9OvL8dWbWFHm+y8w+bGYbzeyImf2HmVVG9q8xsw1m1mpmT5vZxWb2D8BrgC+Fv+i/FB7rZnZGuD3LzL5pZs1m9qyZfSzymf7EzB4ys382s0Nm9oyZrR4m/lnA3wNXu/sP3b3D3Xvd/cfu/pFxfMaPmtlGoCOM5ftD3uffzOyGSOzfCGsqz5vZp80sPZ7rLqVLvxCkFP0t8CrgXMCBHwEfA/4OuBZoAhrCY18FuJm9GLgGeIW77zGzJUASX1SXARcDXcB/AX8C3GRmK4FvAu8A7gfmA7Xufo+ZXQB8y92/PsxrfhGYBZwG1AH3AXuBb4T7XwncBtQDa4FvmNkCL5wn59VAJXDXMX7GdwGXAC3APOBvzGymu7eGX/6XAW8Lj70NeAE4A6gGfgLsBr56jDFICVANQkrRe4C/d/f97t4MfBJ4b7ivl+DL99Tw1/Gvwy/KPqACWG5m5e6+y92fHuE9PhzWQg6bWcs4YrvB3fe4+0HgxwRJDOAK4BZ3/7m797v78+6+bbQXC79w3wn8tbu3ufsu4PORzwvwrLt/zd37CL6Q5xM0Hw1VB7S4e3YcnyfODe6+29073f1Z4L+BS8N9rweOuvtvzOwkYDXwwbC2sh/4V+DyY3x/KRFKEFKKTgGejTx/NiwD+BywA7jPzHaa2XUA7r4D+CDwCWC/md1pZqcwvH9299nho34cse2LbB8FasLtRcBICWk49UCGws+7IO493f1ouFlDoQNA/ST0Hewe8vwOgloFwLvD5wCnAuXA3lyyJag5zDvG95cSoQQhpWgPwZdPzuKwjPBX9rXufhrwVuBDub4Gd7/D3VeF5zpw/TjftwOoijw/eRzn7gZOH2bfSFMmtxDUioZ+3ufH8d45jxA0fV06wjFj+YxD4/0e8DozW0jQtJRLELuBbqA+kmxnuvtZE4hdSpAShBRbuZlVRh5lwHeAj5lZg5nVAx8HvgVgZn9gZmeYmQGtBE1LfWb2YjN7fdiZ3QV0hvvGYwPwFjOba2YnE9RIxuobwPvN7A1mljKzBWZ2ZrjvBYL+hQJhs9F3gX8ws1ozOxX4UO7zjoe7HyG4Vjea2aVmVmVm5Wa22sw+O9HPGDbzPQD8O/CMuz8Rlu8l6C/5vJnNDD/36Wb22vHGLqVJCUKKbT3Bl3nu8Qng00AjsBHYRNAGnrvzZhnwn0A7wS/mL7v7AwT9D58h+EW+j7BzdZyx3A48Duwi+OIb862v7v474P0EbfBHgF8xWCv4N+Ad4V1IN8Sc/ucEv+x3Ag8R/EK/ZZyx5+L4F4IE8zGgmeBX/jXA3eEhE/2MdwBvZLD2kPM+giayrcAh4PsEfSQyDZgWDBIRkTiqQYiISCwlCBERiaUEISIisZQgREQk1rSaaqO+vt6XLFlS7DBERI4bjz32WIu7N8Ttm1YJYsmSJTQ2NhY7DBGR44aZPTvcPjUxiYhIrEQTRDjd8XYz25GbM2eY415hZn1m9o7xnisiIslILEGEs1TeSDDb43LgXWa2fJjjrgfuHe+5IiKSnCT7IFYCO3LLKprZnQSrWm0dctyfAz8AXjGBc0VEJqy3t5empia6urqKHUriKisrWbhwIeXl5WM+J8kEsYD8aYObCBY+GWBmCwhmh3w9+Qli1HMjr7GWYBEVFi9efMxBi8iJo6mpidraWpYsWUIw/+P05O4cOHCApqYmli5dOubzkuyDiLvaQyd++gLw0XBGy/GeGxS63+zuK9x9RUND7J1aIiKxurq6qKurm9bJAcDMqKurG3dNKckaRBPBIio5Cwnn9I9YAdwZ/sepJ5iGODvGc0VEjtl0Tw45E/mcSdYgHgWWmdlSM8sQLEO4LnqAuy919yXuvoRgmuA/c/e7x3LuZPri/U/xqyebk3p5EZHjUmIJIlwX9xqCu5OeAL7r7lvM7Cozu2oi5yYV61d+9TQPPaUEISISlehIandfT7AgTLTspmGO/ZPRzk1Kyox+LYshIpJHI6kBM+jXwkkiUiRf/epXufrqq4sdRgElCIIahPKDiBTLxo0bednLXlbsMAooQQAp1SBEpIg2bdpUkCC2bdvGhRdeyFlnncUb3/hGWlpaALjtttt4+ctfztlnn81rXvOaYcsmw7SazXWigj4IJQiRE9knf7yFrXtaJ/U1l58yk//31rNGPW7z5s289KUvHXje3d3N29/+dr71rW9x3nnncf311/Ov//qvXHfddVx//fVs2LCBTCbD4cOHaWtrKyibLKpBENwfrE5qESmG3bt3U1tby6xZswbK7r77blatWsV5550HwPLly9m/fz/pdJrOzk6uvfZaGhsbmT17dmzZZFENgqCJyVWDEDmhjeWXfhLi+h+2bt2aV7Zp0yaWL19OVVUVmzdv5sc//jFr167lyiuv5M/+7M9iyyaDEgRhE1N/saMQkRNRXP/DggUL2LBhAwA7d+7k9ttv56GHHuKpp55i2bJlXH755WzdupWurq7YssmiBIE6qUWkeDZt2sQ999zDd77zHQDmz5/PL37xC9avX8/LXvYyZsyYwS233EJdXR3XXnstjzzyCNXV1Zx11ll87Wtf46qrrioomyxKEKgPQkSK59vf/nZs+d13311Qduutt46pbLKokxpIpdQHISIylBIEus1VRCSOEgSai0nkRHaitB5M5HMqQaC5mEROVJWVlRw4cGDaJ4ncinKVlZXjOk+d1GguJpET1cKFC2lqaqK5efpP959bk3o8lCDQba4iJ6ry8vJxrdF8olETE+qkFhGJowRBMA6iTyOpRUTyKEGguZhEROIkmiDM7GIz225mO8zsupj9a8xso5ltMLNGM1sV2bfLzDbl9iUZZzqlJiYRkaES66Q2szRwI/AmoAl41MzWufvWyGH3A+vc3c3sbOC7wJmR/Re5e0tSMUZi1TgIEZEhkqxBrAR2uPtOd+8B7gTWRA9w93YfbNupBoryNa27mERECiWZIBYAuyPPm8KyPGb2NjPbBvwU+NPILgfuM7PHzGztcG9iZmvD5qnGid7LrHEQIiKFkkwQFlNW8DXs7ne5+5nApcCnIrsucPfzgdXA1WZ2YdybuPvN7r7C3Vc0NDRMKFDVIERECiWZIJqARZHnC4E9wx3s7g8Cp5tZffh8T/h3P3AXQZNVIkzjIERECiSZIB4FlpnZUjPLAJcD66IHmNkZZmbh9vlABjhgZtVmVhuWVwNvBjYnFWhQg0jq1UVEjk+J3cXk7lkzuwa4F0gDt7j7FjO7Ktx/E/B24H1m1gt0Au8M72g6CbgrzB1lwB3ufk9SsabM6NOaoyIieRKdi8nd1wPrh5TdFNm+Hrg+5rydwDlJxhal6b5FRAppJDWa7ltEJI4SBKpBiIjEUYJAczGJiMRRgkDTfYuIxFGCIBwHoZuYRETyKEGgkdQiInGUINBcTCIicZQggFRKNQgRkaGUIAiXHFWCEBHJowSBmphEROIoQQBpdVKLiBRQgkDjIERE4ihBoHEQIiJxlCDQVBsiInGUINBkfSIicZQg0DgIEZE4ShDk1qQudhQiIqVFCQL1QYiIxEk0QZjZxWa23cx2mNl1MfvXmNlGM9tgZo1mtmqs504m3eYqIlIosQRhZmngRmA1sBx4l5ktH3LY/cA57n4u8KfA18dx7qRRJ7WISKEkaxArgR3uvtPde4A7gTXRA9y93QfbdqoBH+u5k0lrUouIFEoyQSwAdkeeN4VleczsbWa2DfgpQS1izOeG568Nm6cam5ubJxSo5mISESmUZIKwmLKCr2F3v8vdzwQuBT41nnPD82929xXuvqKhoWFCgWrBIBGRQkkmiCZgUeT5QmDPcAe7+4PA6WZWP95zj5U6qUVECiWZIB4FlpnZUjPLAJcD66IHmNkZZmbh9vlABjgwlnMnk8ZBiIgUKkvqhd09a2bXAPcCaeAWd99iZleF+28C3g68z8x6gU7gnWGndey5ScWqcRAiIoUSSxAA7r4eWD+k7KbI9vXA9WM9Nym6zVVEpJBGUqNOahGROEoQBH0Q7mpmEhGJUoIgaGIC1MwkIhKhBAGkw6ugZiYRkUFKEARNTKAEISISpQTBYBOT8oOIyCAlCIK7mEA1CBGRKCUI1EktIhJHCYJgum9QDUJEJEoJgkgfRH+RAxERKSFKEKgPQkQkjhIEkErpNlcRkaGUIIiOgyhyICIiJUQJgsEmJs3FJCIySAkC3eYqIhJHCQJ1UouIxFGCQHMxiYjEUYJAczGJiMRJNEGY2cVmtt3MdpjZdTH732NmG8PHw2Z2TmTfLjPbZGYbzKwxyTjVxCQiUiixNanNLA3cCLwJaAIeNbN17r41ctgzwGvd/ZCZrQZuBl4Z2X+Ru7ckFWOOOqlFRAolWYNYCexw953u3gPcCayJHuDuD7v7ofDpb4CFCcYzrNxcTH3KECIiA5JMEAuA3ZHnTWHZcK4AfhZ57sB9ZvaYma0d7iQzW2tmjWbW2NzcPKFAB/sglCBERHISa2ICLKYs9hvYzC4iSBCrIsUXuPseM5sH/NzMtrn7gwUv6H4zQdMUK1asmNA3vJqYREQKJVmDaAIWRZ4vBPYMPcjMzga+Dqxx9wO5cnffE/7dD9xF0GSVCK1JLSJSKMkE8SiwzMyWmlkGuBxYFz3AzBYDPwTe6+5PRsqrzaw2tw28GdicVKAaByEiUiixJiZ3z5rZNcC9QBq4xd23mNlV4f6bgI8DdcCXwy/prLuvAE4C7grLyoA73P2epGLVOAgRkUJJ9kHg7uuB9UPKbopsXwlcGXPeTuCcoeVJ0TgIEZFCGkmNOqlFROIoQaA1qUVE4ihBoHEQIiJxlCBQE5OISBwlCCKd1MoQIiIDlCDQmtQiInGUINCa1CIicZQggFRKNQgRkaHGlCDCqS9S4faLzOwPzaw82dCmjgbKiYgUGmsN4kGg0swWAPcD7wduTSqoqaa5mERECo01QZi7HwX+CPiiu78NWJ5cWFNr6FxMnT199Pb1FzEiEZHiG3OCMLNXA+8BfhqWJTqP01Qa2sT0ko/fw2VffaSIEYmIFN9YE8QHgb8G7gpnZD0N+GVyYU2tuIFy//Pc4SJFIyJSGsZUC3D3XwG/Agg7q1vc/S+SDGwqaU1qEZFCY72L6Q4zmxku3rMV2G5mH0k2tKmjuZhERAqNtYlpubu3ApcSrO+wGHhvYlFNMc3FJCJSaKwJojwc93Ap8CN37wWmzddpdE1q1SJERAJjTRBfBXYB1cCDZnYq0JpUUFMtOg6it08JQkQExpgg3P0Gd1/g7m/xwLPARaOdZ2YXm9l2M9thZtfF7H+PmW0MHw+b2TljPXcypSIJokfjH0REgLF3Us8ys38xs8bw8XmC2sRI56SBG4HVBIPq3mVmQwfXPQO81t3PBj4F3DyOcydNpiy4DD3ZfnqzShAiIjD2JqZbgDbgsvDRCvz7KOesBHa4+0537wHuBNZED3D3h939UPj0N8DCsZ47maozaQA6ujWCWkQkZ6yjoU9397dHnn/SzDaMcs4CYHfkeRPwyhGOvwL42XjPNbO1wFqAxYsXjxJSvKpMcBmO9mTpVg1CRAQYew2i08xW5Z6Y2QVA5yjnWExZbA+wmV1EkCA+Ot5z3f1md1/h7isaGhpGCSlepixFedroiJmDqbOnj0MdPRN6XRGR49lYaxBXAd80s1nh80PAH49yThOwKPJ8IbBn6EFmdjbwdWC1ux8Yz7mTqSpTFk7Sl5+HLvnir9nZ3MGuz1yS5NuLiJScsd7F9Li7nwOcDZzt7ucBrx/ltEeBZWa21MwywOXAuugBZrYY+CHwXnd/cjznTrbqTJqO7mxBDWJnc0eSbysiUrLGtaKcu7eGI6oBPjTKsVngGuBe4Angu+FEf1eZ2VXhYR8H6oAvm9kGM2sc6dzxxDpeMzJpjvb05fVBZIfpsH746RZ+uX1/kuGIiBTdsUzZHddPkMfd1xNMzREtuymyfSVw5VjPTVJ1RRkdPfk1iF0HjsYe+5UHnqa1K8tFL543VeGJiEy5Y1mTeloNOa7KpDk65DbX7fva8o65b8u+YKxEX/+wtQsRkelixBqEmbURnwgMmJFIREVSnSnjhbYueiJNTDub2we2f/fMQdbe/hhXrFpKX7+T1ZQcIjLNjZgg3L12qgIptqqKMo625Ncgdh8KmpjK00ZXbx8A2/a1ku13sv2qQYjI9DZtlg09VlXlaTp6svREagbPHQwSRLbfqQpHW7d3Zen3oExEZDo7lj6IaaWqIuiDiDYx7WoJEoQ7A+Mj2rqyQQ1CTUwiMs0pQYSqM8FdTNEEsa+1a2A718TU1p2lr79fTUwiMu0pQYSqKtL0O3R0Z/PKl9RVAdAZJoj2rizZPtUgRGT6U4IIVYcT9h06mj/v0uK6YFbzoz1Bgujs7Qs7qZUgRGR6U4IIzQg7oQ939uaVz60qBwZrEEB4m6uamERkelOCCNVUBDWIg+35NYiZM4IE0dUzmCB6+vpVgxCRaU+3uYZOnlUJDN7amjOzsrAGcaijZ3oNIxcRiaEaRGjB7GBg+K4D+bO3zppRmCCy/U5fv/P9x5q0VoSITFtKEKGGmgrK0zbQGZ1TWxlUsjqHlAN8+HuP85NNe6ckPhGRqaYEEUqlbKCZKSrXed3VW5ggALqHKRcROd4pQUScMqtw/sF0KpjVvHOYRKDOahGZrpQgInL9EFFpCxNETBMTQG9Wt7uKyPSkBBGxpL66oGy0GsTQJUpFRKYLJYiI97361IKysvQoNQg1MYnINJVogjCzi81su5ntMLPrYvafaWaPmFm3mX14yL5dZrYpulZ10mZXZfj1/72IL7zz3IGydCq4RF1ZNTGJyIklsYFyZpYGbgTeBDQBj5rZOnffGjnsIPAXwKXDvMxF7t6SVIxxFs2tyrvVtSw1cg1CndQiMl0lWYNYCexw953u3gPcCayJHuDu+939UaA37gWKJdesZDbYB9HVG19T6FEfhIhMU0kmiAXA7sjzprBsrBy4z8weM7O1wx1kZmvNrNHMGpubmycYar7ysFkpk04N1iB6+8ikCy+XJu0TkekqyQRhMWXjaY+5wN3PB1YDV5vZhXEHufvN7r7C3Vc0NDRMJM4CuRpEpiw1eBdTTx9VFemCY3c2d3Dzg09PyvuKiJSSJBNEE7Ao8nwhsGesJ7v7nvDvfuAugiarKZGrNVSUpSgLaxOdvX0Da0ZENT57iH9cv439bV0F+0REjmdJJohHgWVmttTMMsDlwLqxnGhm1WZWm9sG3gxsTizSYSyYPWOgBgFQlSmsQeT09jn/uP4JHnxycpq5RESKLbG7mNw9a2bXAPcCaeAWd99iZleF+28ys5OBRmAm0G9mHwSWA/XAXRaMYi4D7nD3e5KKdah5Myv57NvP5nVnNnD46GD/eVXF8Jdr35Eubn5wJzc/uJP//NBrOWNezVSEKiKSmETXg3D39cD6IWU3Rbb3ETQ9DdUKnJNkbKO57BVB61hb1+Aa1VXlw9cgnnyhLW9bCUJEjncaST2KskgTU0X58JcrmiCGm/lVROR4ogQximgfRNxtrjnRBNGt0dUiMg0oQYwidxcTQHnZSAmifWBbNQgRmQ6UIEYx1hpEc1s3S+qqgMEahLvz3cbdHOksqYHiIiJjogQximgfRHk6buzfoKXhdOG5GsTjTUf4v9/fyCfWbUkuQBGRhChBjCKdjiaIkS/XSTMryaRTAzWIfUeCwXMHO3qSC1BEJCFKEKPIr0GMfLnmVmeoKEsN1CB2HzwKQE1loncTi4gkQgliFHl9ECN0UgPU1VRQUZ4eqEE8c6AD0IR+InJ8UoIYRfQuppE6qQHqwhpEdzg1+K6WIEEcaFcTk4gcf5QgRhGpQIzaVDS3OkNleWpg9blnDwRNTC3t3YnFJyKSFCWIUYTzQQFQO4YEUVGWpru3H3enuS1IDNEaxLMHOugeZvlSEZFSogQxDrWV5SPur6sJahDd2T66evvp6euntqKMtu4sXb19dGf7eO3nHuDK2xr5wWNNUxS1iMjEKEGMw3hqELnBcac1BGMjDnT00NsXrJf066dauPZ7j7N1T2uyAYuIHAMliHGYOUqCqChLD/RB5BLE6Q3BrK4tbd309eUvqNfapRHWIlK6lCDGYbQmJmCEGkQ3vf35t7u2agoOESlhGsE1DsM1Mf3pBUv5yzcuAyioQZw2UIPooa8/vwbRottfRaSEqQYxDsPVIKoyaWbNCPYNV4No6eimd8iAOd3+KiKlTAliHKqHWZM6OlZiaA3i5JmVVGfSw9QglCBEpHQlmiDM7GIz225mO8zsupj9Z5rZI2bWbWYfHs+5xRAdE/GtK17JmnNPKSivKM+vQdRWllNXUxH0QQzppNYIaxEpZYklCDNLAzcCq4HlwLvMbPmQww4CfwH88wTOLapVy+pZPDdY/yE6X1NFWVCDaO3spbayjHTKqK/J0NLeTXZIJ3WzahAiUsKS7KReCexw950AZnYnsAbYmjvA3fcD+83skvGeWyw/+fNV/PaZgwADTUb5TUxp3IPmo1y/RF1NBbsPHiXbpyYmETl+JNnEtADYHXneFJZN6rlmttbMGs2ssbm5eUKBjsdLF8ziilVLAch1KeQ1MYUzvu5vG0wQ9TUVtLT3kO0vbGJyd279r2e06pyIlJwkE0Tc8mseU3ZM57r7ze6+wt1XNDQ0jDm48agoSw0sJzrkvQFIDemDgGBgXO622PqaDAc7uunJDjYxZdIp2ruzPN3cwSd+vJV7t+xLJHYRkYlKsompCVgUeb4Q2DMF5066LZ/8/djy/33haTzd3MG7Vy4eKKsMaxAt7d0Dt7jOm1lJv8O+1q6B4xbOmcHOlo6BVefaurJJhS8iMiFJ1iAeBZaZ2VIzywCXA+um4NxJV5ZOURazFkR9TQVf/+MVzKoaHB9RXRHk3NauLFWZYPvkmZUANB06OnDcgjkzAHj+cFDWrgQhIiUmsRqEu2fN7BrgXiAN3OLuW8zsqnD/TWZ2MtAIzAT6zeyDwHJ3b407N6lYJ1MuQQTbQXPT/Fm5BNEJBE1Wr1lWz6+fauH5w0ENoqNHCUJESkuiU224+3pg/ZCymyLb+wiaj8Z07vGgpmJwMF2uBnHSzPwE8b2rXs3BjmAMxPNhmZqYRKTUaCT1JMurQYQjr+uqM5SnjefDJqayVGpg2o5cs1N7txKEiJQWJYhJVp0ZTBBVYbJIpYx5tZXsDmsL5WkbmDr8+cNBWUd3lrau3oGaBcA3HnqGP7310akKXUQkjxLEJKuJqUFA0A+Ru801nbKBGkQuQbR3Zfm7uzdz/qd+zs+3vgDA47sP82g4KE9EZKopQUyyaBNTVaQ20VBbMbBdnk4NjJEIh1LQ3p3lsecOAfC9xt0DZW3d2bxJ/j537zb+5q5NicUvIpKj9SAmWaYsRSadoqevf+AuJoAZ5YPb6ZRRlUmTTtnAl397d5bOnqCGkRsvkeuXaO/KDtxK27jrEIePatS1iCRPNYgE5BJDtAZRUT54qcvShpnlLUB0qKNnYG6mveHgudzYiOjSpEd7+nRLrIhMCSWIBOSamaI1iIqywe2yVHDZowmiLawtLJg9g5b2YFqOXCKIztPU0Z3lUEcPq//t1zy8oyW5DyEiJzwliATkOqrzahBl+TUIgNqKwhXqzls8G3d4obUrtgbR0ZOlo6ePJ/a28njTkUTiFxEBJYhEDNQghksQ4fzgq5bVA/kd2Ocumg3Ap36ylQPhLa+tnVl2HzzK1d/+77xFhlq7evOSh4jIZFIndQJyCaIq0sSUyUsQwfbfvOUlfOC1p3Pvln1c98PgzqTzFs8B4L7wVlcIEsEn1m3h/m37897n9kee5Vu/eZY/OPsUltZXsfql81k4Z0be9OMiIhOlGkQCctNt5Ncgon0Qg1/gc6ozAxP3rVw6l5fMry14vdbO3oK1JCC8DbYryw8ea+L23zzLhZ/7Jfdu2ceO/e2T9llE5MSlGkQCqjNlmEFl5M6l6F1MqVT+L/xVZ9Rz319dyLJ5NZgZv/nrN/Cqf7p/YP+nf/rEiO/X09fP7oPBgLu/+9EWsn39rLtmFbOqyplZWdjPISIyFqpBJGBuTYY5VZnYlebimBkvOql24PiTw9lfJ6K5rZtDR3v5o688zBfvf4pnWjom/FoicmJTgkjA/7nwdG6/YmVeWbSJaSo0t3Wz7vE9XPTPD7Bh92EOH+0Z/SQRkQg1MSVgbnWGudWZvLLMCDWIOGaD03BM1AutwcC7D3zrMebVVnDr+1dSW1kWu/iRiMhQ+qaYIiM1McWpCTu4r33Ti3jZglnH9N57j3SxY387r/3cL/nO756js6fvmF5PRE4MShBTZLxNTLlbZdecu4DLXrFolKNH19HTR2tXlq/9+hle/umfq8lJREalBDFFoncxjcXVrz8DCDq8o9OGH6vnDh7laE8f/+sbv+Vjd2tWWBEZXqIJwswuNrPtZrbDzK6L2W9mdkO4f6OZnR/Zt8vMNpnZBjNrTDLOqTDeJqb3vupUdn3mEmoqyoUwJPMAAAyJSURBVAYGz02mzc+38sttzVx5WyNPvtA26a8vIse/xBKEmaWBG4HVwHLgXWa2fMhhq4Fl4WMt8JUh+y9y93PdfUVScU6V8XZSRy2tr+apf1g9idEEnj/cyX8+8QK3PryLHzzWNOmvLyLHtyRrECuBHe6+0917gDuBNUOOWQN80wO/AWab2fwEYyqaY73NtTyd4n2vPnWSosl3x2+f49rvPc6mpiOa20lEBiSZIBYAuyPPm8KysR7jwH1m9piZrU0syiky3iamOJ/8w7N45p/ewg3vOo/PvuNsIKhdTNbUS2/90kN88M4NHNV6EyJCsgki7mtr6J39Ix1zgbufT9AMdbWZXRj7JmZrzazRzBqbm5snHm3CJiNBmAULDf3hOadw2YpF7PrMJVy2YhGvfVEDkD+1x0T9Ytt+ln/8Xj57zzaebtacTiInsiQTRBMQvT9zIbBnrMe4e+7vfuAugiarAu5+s7uvcPcVDQ0NkxT65KsoT2Yk9Qdedzq3vn8lH/n9F/Pnr19Gedo4rb564M6nstT4qhdzqzOcWlfFlx94mjd8/ldc9tVH+F7jbt0WK3ICSnIk9aPAMjNbCjwPXA68e8gx64BrzOxO4JXAEXffa2bVQMrd28LtNwN/n2CsicskPHr56ovOwN25bMUituw5QltXlht/uYMzT67l/if2U19bQV+/89zBoyycM4O2rixHOns5Y14Nc6szfPTiF1OWSnHWKTNJp4wnX2jnge37+fZvn+Mj399IWcp45Wlz+b3T6/m90+t42YJZGpEtMs0lliDcPWtm1wD3AmngFnffYmZXhftvAtYDbwF2AEeB94ennwTcFU5eVwbc4e73JBXrVChPJ79Gg5nRUFvB6148D4C3nnMK2b5+Dh3t5UhnL9UVaR7ffZjzF89hf1s3C+fMoCpTFnuH1YtPruXFJ9ey9sLT2PT8EX62eR+/eGI/n7t3OxCsmrdiyRzOWzSHcxfP5tyFs5lVpZljRaYT82Od8KeErFixwhsbS3fIxJLrfgrArs9cUuRIJu5Aeze/2XmQh59u4XfPHGRHc/vAnFGn1Vdz7qLZnLNoNi9dMIszT64dGBEuIqXJzB4bbiiB/vXKuNTVVHDJ2fO55OzgbuS2rl42Nh1hw+7D/M9zh3nwqRZ++D/PA8GEg6fOreIl82fykvkzOfPkWl4yf6ZWvRM5TihByDGprSzngjPqueCMYH1td2fPkS62PH+EJ/a2sW1fK0/sbeVnm/cNnFNTUcbpDdWc3lDD6fNqBrZPras+pgGFIjK5lCBkUpkZC2bPYMHsGbz5rJMHyju6s2zb18YTe1t58oU2djZ38MjOAwO1DYB0ylg8t4ql9dUsnlvForlV4d8ZLJpTpeYqkSmmf3FT6LSGas5bNPnzKh0PqivKePmpc3j5qfmfv707yzPNHexsaefp/e083dzB083t/HbnATqGTEteX5Nh4ZwgaSycM4P5s2cwf2YlJ88KHnOrMgXLuYrIxKmTWkqSu3PoaC+7Dx7luYNH2X3o6OD2wU72HO4k25///91MOsVJsyqYP3PGQNKYV1tBQ20FddUV1NVkqKvJMLcqc1zcotvf72T7nWx/P719XjjMdAIsFYxONbPwLxg2MBrfDFID+yLHqM9o2lIntRx3zGxgZb5zFs0u2N/X7xxo72bvkS72Huli35FO9rV2s+9IJ3uPdPF402Hu2dJFT7Y/5rVhTlWGuupMmDQqmFNVzszKcmbOyP0tY9aMaFkZVZkyKspS46qldGf7BsacHOzo4UB7Dwc7ejjY0c2Bjtx2UH6go5u2rmyQFPr66S/B325BQgmTSJhcwv/lJZvcMYP7BsuHJp64c3MJaSBhDTmXIc9zx1Dw+pHXzXuvwvfGIJV3/NDkmXuv/HPJizs4J5Uaem1i3rvgM+e/d+51UwXHDV6bXHlNRRl/9aYXTfp/byUIOS6lU8a8mZXMm1nJOcOsp+TuHOns5UD4BdzS3s2B9m5awi/jXNnWPa0c6eyltbO3oFYSJ5NOUVGWoqI8RUVZmkxZCnen34PE5e709DltXb10xySonJqKsoEkOH9WJWedMpOZM8opSxvlqVTwN50inTLKUhZ84R4DD6+JOzi5v+Q9z123fqfgONzzjo8egw99/WBff/iiPuTcvPceIabc6/bnva6H8UTOCd8rdy4UHp8fT/65ue3gM/XHnhu9BgPxRI4b+rr9eXEOcw18mPLI5+gfekzM69bVVChBiIyHmTG7KsPsqgynj2EWFnens7eP1s7gF39rV5A0Wrt6OXK0l87efrqzfXSFf7uz/XT39tPT108q/KUbPKAsnWJmZRm1lWUDtZI51YO1ljlVGSoTmn5FZLIoQYiEzIyqTNCUdPKsymKHI1J0pd9TJyIiRaEEISIisZQgREQklhKEiIjEUoIQEZFYShAiIhJLCUJERGIpQYiISKxpNVmfmTUDz07w9HqgZRLDSUKpx1jq8YFinCyK8diVSnynunvsXAPTKkEcCzNrHG5Gw1JR6jGWenygGCeLYjx2pR4fqIlJRESGoQQhIiKxlCAG3VzsAMag1GMs9fhAMU4WxXjsSj0+9UGIiEg81SBERCSWEoSIiMQ64ROEmV1sZtvNbIeZXVfseHLMbJeZbTKzDWbWGJbNNbOfm9lT4d85UxzTLWa238w2R8qGjcnM/jq8rtvN7PeLGOMnzOz58FpuMLO3FCtGM1tkZr80syfMbIuZ/WVYXjLXcYQYS+k6VprZ78zs8TDGT4blpXQdh4uxZK7jqIL1YE/MB5AGngZOAzLA48DyYscVxrYLqB9S9lngunD7OuD6KY7pQuB8YPNoMQHLw+tZASwNr3O6SDF+AvhwzLFTHiMwHzg/3K4FngzjKJnrOEKMpXQdDagJt8uB3wKvKrHrOFyMJXMdR3uc6DWIlcAOd9/p7j3AncCaIsc0kjXAbeH2bcClU/nm7v4gcHCMMa0B7nT3bnd/BthBcL2LEeNwpjxGd9/r7v8dbrcBTwALKKHrOEKMwylGjO7u7eHT8vDhlNZ1HC7G4RTl38xITvQEsQDYHXnexMj/EKaSA/eZ2WNmtjYsO8nd90LwjxiYV7ToBg0XU6ld22vMbGPYBJVrdihqjGa2BDiP4JdlSV7HITFCCV1HM0ub2QZgP/Bzdy+56zhMjFBC13EkJ3qCsJiyUrnv9wJ3Px9YDVxtZhcWO6BxKqVr+xXgdOBcYC/w+bC8aDGaWQ3wA+CD7t460qExZcWKsaSuo7v3ufu5wEJgpZm9dITDSynGkrqOIznRE0QTsCjyfCGwp0ix5HH3PeHf/cBdBFXNF8xsPkD4d3/xIhwwXEwlc23d/YXwH2o/8DUGq+1FidHMygm+eL/t7j8Mi0vqOsbFWGrXMcfdDwMPABdTYtcxLsZSvY5xTvQE8SiwzMyWmlkGuBxYV+SYMLNqM6vNbQNvBjYTxPbH4WF/DPyoOBHmGS6mdcDlZlZhZkuBZcDvihBf7osi520E1xKKEKOZGfAN4Al3/5fIrpK5jsPFWGLXscHMZofbM4A3AtsoresYG2MpXcdRFbOHvBQewFsI7tJ4GvjbYscTxnQawd0MjwNbcnEBdcD9wFPh37lTHNd3CKrEvQS/dq4YKSbgb8Pruh1YXcQYbwc2ARsJ/hHOL1aMwCqCZoONwIbw8ZZSuo4jxFhK1/Fs4H/CWDYDHw/LS+k6DhdjyVzH0R6aakNERGKd6E1MIiIyDCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghAZBzPri8zCucEmcQZgM1tikVloRYqtrNgBiBxnOj2YOkFk2lMNQmQSWLB+x/Xh/P+/M7MzwvJTzez+cGK2+81scVh+kpndFa4V8LiZ/V74Umkz+1q4fsB94QhckaJQghAZnxlDmpjeGdnX6u4rgS8BXwjLvgR8093PBr4N3BCW3wD8yt3PIVi/YktYvgy40d3PAg4Db0/484gMSyOpRcbBzNrdvSamfBfwenffGU50t8/d68yshWAqhd6wfK+715tZM7DQ3bsjr7GEYEroZeHzjwLl7v7p5D+ZSCHVIEQmjw+zPdwxcboj232on1CKSAlCZPK8M/L3kXD7YYJZggHeAzwUbt8PfAAGFpWZOVVBioyVfp2IjM+McIWwnHvcPXera4WZ/Zbgh9e7wrK/AG4xs48AzcD7w/K/BG42sysIagofIJiFVqRkqA9CZBKEfRAr3L2l2LGITBY1MYmISCzVIEREJJZqECIiEksJQkREYilBiIhILCUIERGJpQQhIiKx/j/ng99WC5QQwgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZwcdbnv8c/TM0kmG1sIAbIjwSzsjJALiCgioCgqKqBXgy8VULgueM8BxItXvYp6jvcoF5ATPRzgHhRZFBGR5SC4sBgChiQjSxYQhgAZEtaYbaaf80dVd9d0qnu6Z7rSNV3f9+uV13RXV1f/atJTT/1+z28xd0dERLIr1+wCiIhIcykQiIhknAKBiEjGKRCIiGScAoGISMYpEIiIZJwCgYhIxikQSEszs3vN7GUzG5XAsc3MPm9my81sg5l1m9kNZrZfoz9LJEkKBNKyzGwG8FbAgfcl8BE/BL4AfB7YBdgHuBl4T70HMrP2xhZNpHYKBNLKPgE8CFwFLIi+YGZTzewXZtZjZuvM7NLIa58xs8fM7HUz+6uZHVx+YDObBZwNnObuv3P3ze7+d3e/1t2/E+5zr5l9OvKe083sT5HnbmZnm9kKYIWZXWFm/1z2Ob8ys3PDx3ua2U1hmZ8ys8834HckokAgLe0TwLXhv+PMbBKAmbUBtwJ/A2YAk4Hrwtc+DPzv8L07ENQk1sUc+xig290XDbGM7wcOA+YCPwVOMTMLy7Iz8C7gOjPLAb8GHg3LewzwRTM7boifL6JAIK3JzI4EpgPXu/vDwCrgo+HLhwJ7Av/g7hvcfZO7F+7UPw18z90f8sBKd/9bzEdMAJ5vQFEvdvf17r4R+CNBM9Zbw9c+BDzg7muAtwAT3f0b7r7F3VcDPwZObUAZJOMUCKRVLQDudPeXwuc/pdQ8NBX4m7v3xrxvKkHQGMg6YI8hlxKeLTzwYAbI64DTwk0fJajNQBDU9jSzVwr/gK8AkxpQBsk4Jaik5ZjZaOAjQJuZvRBuHgXsZGYHEFx8p5lZe0wweBZ4Uw0fczdwmZl1uvviCvtsAMZEnu8es0/59L8/A+40s+8QNBl9IFKup9x9Vg1lE6mLagTSit4P9BG0ux8Y/ptD0PTyCWARQbPOd8xsrJl1mNkR4Xt/AvxPMzsk7B66t5lNL/8Ad18BXA78zMyONrOR4XFONbPzw92WAB80szFmtjfwqYEK7u5/AXrCctzh7q+ELy0CXjOz88xstJm1mdm+ZvaWwfyCRKIUCKQVLQD+3d2fcfcXCv+AS4GPAQa8F9gbeAboBk4BcPcbgG8RNCW9TtAddJcKn/P58JiXAa8QNCl9gCCpC/AvwBbgReBqSs08A/kZ8M6wDITl6gvLfCDwFPASQbDYscZjilRkWphGRCTbVCMQEck4BQIRkYxTIBARyTgFAhGRjBt24wh23XVXnzFjRrOLISIyrDz88MMvufvEuNeGXSCYMWMGixdXGr8jIiJxzCxuqhRATUMiIpmnQCAiknEKBCIiGTfscgQi0nq2bt1Kd3c3mzZtanZRhr2Ojg6mTJnCiBEjan6PAoGINF13dzfjx49nxowZhOvyyCC4O+vWraO7u5uZM2fW/L7EmobM7EozW2tmyyu8bmZ2iZmtNLOlccsBikg2bNq0iQkTJigIDJGZMWHChLprVknmCK4Cjq/y+gnArPDfGcCPEiyLiKScgkBjDOb3mFjTkLv/wcxmVNnlJOCacFWmB81sJzPbw90bsfzfoLg7Nz7czfsO3JNR7W0AbO7t41dL1jCqPcfR++zGjmNGFPe94eFu3n/gZEa294+na1/bxKPdr3Ls3Enctux52nKl/5hNW/vY2ueMG9VO4f/L3XltUy9jRrYxoq10rC29eTZt7WN8Rztb+pyVL76e8G9AZPt517zd2XeyZtFOg2bmCCYTWaaPYE74ycSsA2tmZxDUGpg2bVpiBbrzry/yDzcuZWXPG1xwwhwA/t/dK7n0npUAHLn3rvzHpw8D4NdLn+cfb1xK98sbOffYffod59SFD7L6pQ0suvAYPnftIw0to26apBW4w8qeN7j8Y4c0uyhCcwNB3CUtdnEEd18ILATo7OxMbAGFVzduBWDdG1uK29ZtKD1+/IXXY/bdvM1xnl63AYDNW/MNLd/Rb57IVZ88tKHHFGmG43/wB3r7tBZKWjRzHEE3wULhBVOANU0qCwD5fPDFjLTk0B558vqmrdu7SP1Em41Ehrs0h4FzzjmH6dO3WaG0ZTXzynIL8Imw99B84NVm5gcAwjhALtL+Em3f39xb3x1+vsGrv5XnIkSGKzMjrYsjPvXUU9x7771s2bKF119PLi/X19eX2LHrlWT30Z8BDwBvNrNuM/uUmZ1lZmeFu9wGrAZWAj8GPpdUWWrl4T2KVQgEdR+vwV/0kaoRSIsI/qrSGQm+9rWv8dWvfpW5c+fS1dVV3L5mzRpOPvlkDjroIGbPns2iRYtitwHMnz+fp59+GoDnnnuOzs5OAD784Q9z7rnn8va3v52LL76YG2+8kfnz53PAAQdw5JFH0tPTU/Gzli1bxhFHHFEszyOPPMI73vGOhpxzkr2GThvgdQfOTurzB6NQI4gmZIcSCBpeI1AgkBZhVvlG6eu/7uKva15r6OfN3XMHvvbeeQPu19XVxfLly7n66qv505/+RFdXF/Pnz6e3t5cTTjiBb33rW5x44on8/e9/p6+vjyOPPHKbbe7OM888U2xaWrp0Kfvttx8Ay5YtY86cOdxzzz0ArFu3jg996EPBeX/961x//fWceeaZsZ81duxYVq1aRV9fH21tbXz5y1/m+9//fkN+PxpZHOG+bY5gSDWCoRaojJqGpFWYpbM+cOGFF/LNb34TM2POnDksXx6Mh7355puZM2cOJ554IgBjxozhxhtv3GYbwIoVK5g5c2axZaEQCDZt2sT69eu56KKLip931VVX8fOf/5zNmzfzwgsv8O1vfzv2swrmzZtHV1cXK1asYNq0aRx8cGPG4SoQRBSSxW2RKkF7WSBw920GbPxxRQ9tOWPta5uZu+cO/fZtJCWLpVUYVrHGXMudexL+/Oc/c8cdd7BkyRLOPvtsNm3axP777w/AkiVLmD9/fr/947ZBcNdfqAEALF68mDPPPJOuri4OO+ww2tuDy+4111zDokWL+N3vfse4ceM46qijmDdvHrfeemvscSFocrrvvvu4/PLLuf322xt16goEUaWmodKFPld20d/cm6djRFu/bR//t0X9nueKA8UaWz7VCKRVVGsaapavfOUr3HrrrRxzzDEAvPjiixx00EEA7L777jz66KPFfXt6emK3TZw4kfXr1zN69GgAHnvsMX7zm99w6aWX8tvf/rYYWCAIGIcffjjjxo3jpptu4v7772e//fZj8eLFsceFIBCcfvrpnH322UyePLlh564rS0ThDsUqdB+tVSGQ5BueLNZoMmkNRrqahu666y42b95cDAIAkyZNYsOGDaxfv57TTz+dF198kXnz5nHggQfywAMPxG4DOO6447j77rv5yEc+wg033MCECROYNGkSy5Yt6xcIFixYwCWXXMJb3/pWnnzySfbaay/Gjh1b8bgAs2fPZtSoUZx33nkNPX/VCCI8pvtobpumodqPp+6jIhWYNbzpdCiOPfZYjj322G22v/rqq8XHt9xyyzavx22bOnUqS5cuLT4v5ATKE7v77rsvq1atKj6/4IILABg3blzscQF++MMfcvHFFzN27Nhqp1M3XVkiCt1HKw0og/ou7goEIvFUt63PqlWrmD17Nhs3bmTBggUNP75qBBEDDSiD+qqz+cbOMKFksbSMNOYI0uxNb3oTjz/+eGLH15UlopQjqBwI6rnL71ONQCRWkCNQJEgLXVkiSjmC0rZtu4/Wfry+BmeLVSOQVpHmKSaySFeWiMI4gmivoW2Txc3LEYxSjUBahLHtTVWaksfD2WB+j7qyRMTlCIz01Ag0xYS0imBkcenvo6Ojg3Xr1ikYDFFhzeKOjo663qdkcURcjqC8HbOuXkNqGhKJZfRvGpoyZQrd3d3FSddk8Do6OpgyZUpd71EgiCh8L6OtQeXX8nou7UoWi1RQNtfQiBEjmDlzZtOKk3W6skSUJp2z6MZ++9TVa6jBNYLy6S5EhiuDdA0tzjgFgoh8MRCUtpV/V5s5slikVZTnCKS5FAgi4iadK2/nr+XaXnh3X4MHlIm0ivIcgTSXAkFE3KRz5d/VZjYNibSKtK5HkFUKBBFxk84NJVnc6KahIayRI5IquZRNOpd1CgQRhWagfjmC8mRxHXf5jQwEB03biUNn7tKw44k0k2oE6aJAENHolpxGNg1943370q5xBNJCVCFID11ZIgp38NHrd/ldfS13+YWWpUbWCHL6n5IWYmaqEaSILi8RhWag6PV72/lQaj9eI3sNtSsSSAsJxhEoFKSFri4Rha9l9E6+vHWnWVNMqFVIWolyBOmiy0tE3EW+fNBLs6aY0KhiaSVxs49K8ygQRBRu4KM9hYYyVW4jk8VqGpJWEuQIFAnSQpPORXgkWfy92x9n49Y+JowdWbYP3LfyJf7XzcsHPJ6SxSLxVCNIF11eIgprDLvD5feu4t/ve7r4ZX3vAXsG+zg1BQFobI2gfMlMkeFMaxaniwJBRKn76LbJ4uPmTQLqmyiroYFAOQJpKeo+miYKBBGF5G70C1q48BcuxPmyLqHVvsyNbBpSjUBaSVAjUChICwWCiHwpW1zctnFLH1CakbS+GkHjyqZAIK1EOYJ0USCI6Au/mNEWnXUbtgCl+YfKv7zVLs+NTRYrEEjr0HoE6aJAEFGoEUS/oOs3bCFnpX78zVq8vl2BQFqI1iNIFwWCiN4wAdCvRvDGZsysrvmDLKwnNDIQaECZtBKNLE6XzAeCM///YuZddDure94otun/6N5VxdfXldcI6ji2ksUi8ZQsTpfMDyi7o+tFALpf3hh74d7cmw/u8CM1glq/vuo+KhLP1H00VTJfIyjIu9Mbc+HO5x0GmSNo5PoGShZLSzHUNpQiCgShvHvsbKG9eSdXqhDUVZ1t9FKVIq1CcSBdFAhC+Xx8U05f3jFsUDkCLV4vEs+0ZnGqKBCE8u6x00b3FWoEhRxBHRd3BQKReKoRpEuigcDMjjezJ8xspZmdH/P6jmb2azN71My6zOyTSZanmrxXqRFEuo82q9eQSCvRpHPpklggMLM24DLgBGAucJqZzS3b7Wzgr+5+AHA08H0zG0kTuHt8IHDHKI0NqOfirhqBSLyc1iNIlSRrBIcCK919tbtvAa4DTirbx4HxFkzkMw5YD/QmWKaK8h5/kQ9qBKUpJsq/u9tr0jmRVqK5htIlyUAwGXg28rw73BZ1KTAHWAMsA77g7ttM1WZmZ5jZYjNb3NPTk0hh+yrUCMLPL046V75LtS9zo2oEv/zc4Q05jkhqqGkoVZIMBHEd38v/648DlgB7AgcCl5rZDtu8yX2hu3e6e+fEiRMbX9LgMyr2+89FagTld/nVej40avbRg6bt3JgDiaSEVZ2uUba3JANBNzA18nwKwZ1/1CeBX3hgJfAUMDvBMlVUaRwBUDVZHBsH6piXSCSLNMVEuiQZCB4CZpnZzDABfCpwS9k+zwDHAJjZJODNwOoEy1RRPl/5wm0QaRryfvcy1S72ShaLxFP30XRJbK4hd+81s3OAO4A24Ep37zKzs8LXrwC+CVxlZssIvhvnuftLSZWpmkrjCCAIArkKVYJqX+ZKxxPJOnUfTZdEJ51z99uA28q2XRF5vAZ4V5JlqFXeveIX0yJTTJTXAKrVCOoZfCaSJcGkc/r7SAuNLA5VGlAGVF2YZnv0GhJpNaoRpIsCQSjvXiVHUHlhmmoXeyWLReJpYZp0USAI5b3yHYpF5hoq36VaHqARFYJFFx4z9IOIpI6WqkwTBYJQpSkmIGgWKvR7Lu/yFpcHKOQThto0NH5UO7uN7xjSMUTSKLixUiRIi0wHguhFvC9fuWkIIBf+psp3UdOQSP00xUS6ZDsQRL6JleYagiAIlCadq3yMckoWi8RTjiBdMh4ISo+rTTERLEwT7kftyWIFApF4hhamSZOMBwLv97ha99FSr6H+r/VV+S6raUgknmoE6ZLpQBC9TldrGorOPlpLsrhANQKReIYGXKZJpgNBeY2g4qRzRBevr3yMctVqC7V459xJQzuASEqZmWoEKZLpQBAdA5DPV8kRREcW15EjGModzymdU/nuyfsP+v0iqadIkBqZDgTRJXDyXnlwWHQa6nzZGgNJ9RqatGMHI9sz/d8jLUw5gnTJ9JWmvGmoUi+GfnMNlb2mcQQi9VOvoXRRICg+rnxRj66mtM1cQ5p0TqRuqhGkS8YDQelx1XEEBrlc/GRDVXsN6Y5HJJZGFqdLxgNB6ZvYW+WCbmYV1yOIu+sv5RP0TReJk8tpPYI0USAI9VZZad6onCOo3n1UX3SROKoRpEvGA0HpcbUaQS7H4NYjqBxbBnTYzF0G/2aRtFOOIFUSXaoy7aJNN71Vsr7RhWm2mX20wd1HH/vG8TjOmJGZ/q+RFmeKBKmS6auN11gjCNYsrn+KicF0Hx09sq3u94gMN0GvIUWCtMh001BfrTkCi84+WvkY5TSOQCSecgTpkulAUHOvIShOOldeA6iWB9A4ApF4GkeQLpkOBNFmnq1VagTByOLwPWWvaT0CkfppZHG6ZDoQRK/T1S7aFlmzOO/9g0HSi9eLtCLVCNIl44EgWiMYoGmouGax1iMQGSrlCNIl24Eg0hrUV6WxP2dWGlBWQ/fRQu1BA8pEKjAbeB/ZbrIdCGpMFmNUnGKiavdR1QhEYpUWetLfSBooEIQGThZXmmKi8vFVIxCJV2mApjRHxgNB6XGtI4trmWKiMFBGX3KReMUBmk0uhwQyHghqaxqKzjVUS45ALUIi1ZX+nvTHkgaZDgTRL2HV7qNYXVNMKDcgUl0xR9DUUkhBpgNBNC1QLUdg0QFlNdQIlBsQqU45gnTJdCCoa2EaKw0oi4r7IuvLLVKdFTtf6I8lDRQIQgMvTBM8diovcj9UndN3TuS4Immlm6Z0yHQgqHUa6pzRr0aQVArgxs8ensyBRVJG48nSJdOBoH+NoHrTUPATcNf00iJDVJq7S39LaZDpQBDtKdRbZYoJi/zMu6qzIkOlZHG6ZDoQRL+EcZPOFfIChRpBzgxHNQKRoVL30XRJNBCY2fFm9oSZrTSz8yvsc7SZLTGzLjP7fZLlKVe4oLflLHYcQXsuF5aR4s8gR6Cvr8hQlCZx1N9SGgy4ZrGZjQU2uns+fJ4DOtz97wO8rw24DDgW6AYeMrNb3P2vkX12Ai4Hjnf3Z8xst8GfSv0K1/62nMWOI2jLGfT1rxl4gslikawoNg01txgSqqVGcDcwJvJ8DPCfNbzvUGClu6929y3AdcBJZft8FPiFuz8D4O5razhuwxTu7EdUqBG0hRGgkNja0pvnit+vYtOWvu1XSJEWpgpBOgxYIyC4+3+j8MTd3zCzMdXeEJoMPBt53g0cVrbPPsAIM7sXGA/80N2vKT+QmZ0BnAEwbdq0Gj66NoVqaXtbjg0xF/dok1DU65t7h/zZu44bWaxhgPOFY2YN+Zgiw4WpSpAqtQSCDWZ2sLs/AmBmhwAba3hfXE/h8v/2duAQ4BhgNPCAmT3o7k/2e5P7QmAhQGdnZ8O+OoXWoPZcfKfmXCRJ3GiLv3psw48pMlyUksWKBGlQSyD4InCDma0Jn+8BnFLD+7qBqZHnU4A1Mfu85O4bCALOH4ADgCfZDqLJ4ji5aL9REWkYdR9NlwEDgbs/ZGazgTcTXBIfd/etNRz7IWCWmc0EngNOJcgJRP0KuNTM2oGRBE1H/1JH+YekEAiaUSMQyTJ1H02XAZPFZnY2MNbdl7v7MmCcmX1uoPe5ey9wDnAH8Bhwvbt3mdlZZnZWuM9jwO3AUmAR8BN3Xz7406lP4W6kvS3+11DMEWyn8ohkhan7aKrU0jT0GXe/rPDE3V82s88QdPusyt1vA24r23ZF2fN/Av6ptuI2VrFG0BZ/qe83tYSINIxyxelSS/fRnFnpUhiODxiZXJG2n0KX0UpNQ21qGhJJRGnx+qYWQ0K11AjuAK43sysIAvhZwG8TLdV24sUBZfHxMKemIZFkaD2CVKklEJxH0If/swTXxL8Q9Bwa9gZKFpeahoYeCg6YsiN77DiaXA6OmjVxyMcTGc6Kf1GKA6lQS6+hvJk9COxF0G10F+CmpAu2PUSnmIhTqCgMJg4s+G/TufqBvxWf/+vHO9l9x476DyTSgpQjSJeKgcDM9iHo8nkasA74OYC7v337FC15xSkmKiSLS91H6z/25t7K01qLZF1h2hblCNKhWo3gceCPwHvdfSWAmX1pu5RqOyk1DVXKEfSfa6gem7ZqPiKRSko1AkWCNKjWa+hk4AXgHjP7sZkdQ4vlTfP5gbqP9v9Zj01bVSMQqUS9htKlYiBw91+6+ynAbOBe4EvAJDP7kZm9azuVL1ED5giGkCze3KsagUglyhGky4DjCNx9g7tf6+4nEswXtASIXWRmuBl4iongp2oEIo1VyhEoFKRBLd1Hi9x9PfCv4b9h56r7nuL6xd3F5+s2bAYq1wgK4wtaqj1MJA006Vyq1BUIhrv/fGwtz72ykbfM2AWAPXcazXv2G8PRb55IXx4m7TCKDx48mXse72Fzbx/H77sHP/7Dak7cf08AvvPB/VjV8wbdL29k2oQx4LD6pQ3s0DECxxk/qp1czmgz45NHzmTl2jd46Kn15HLGpB1GNfPURVJFN1fpkqlA4DizdhvHTxZ0bvPaUfuUBnkdMn2X0uOPH1J8fOqh9S2KM3mn0bxtHw0eEylXmnSuyQURIOHF69Mmn9e8QSJpoIVp0iVbgcBddVKRFNDCNOmSqUDgPrhRwiLSWLnipHOSBpkKBHl3NQ2JpEDhzzCvKkEqZCoQOMoRiKSJ4kA6ZCoQ5N212phICpRG6ysSpEHGAkFj1hYQkaHRXEPpkqlA4O5KFoukgOYaSpdMBQIli0XSQesRpEumAoG6j4qkg9YjSJdMBQLlCETSQTmCdMlUIHB3DSwWSQGNLE6XTAUC5QhE0qIwsliRIA0yFQjcocLyxCKyHalGkC6ZuiwGA8pUIxBpNv0VpkumAkHQa0hfQZFm03oE6ZKpQJBXslgkFbQeQbpkLBBoHIFIGihHkC6ZCgSOeg2JpIGmmEiXTAWCfF4DykTSoDTFhEJBGmQqEGjSOZGUUI0gVTIVCIIpJppdChHRFBPpkrFAoByBSBpoYZp0yVQgcJQjEEkD1QjSJVuBQDkCkVRQr6F0yVQgUI5AJB1yGlmcKokGAjM73syeMLOVZnZ+lf3eYmZ9ZvahJMujHIFIOpSahhQJ0iCxQGBmbcBlwAnAXOA0M5tbYb/vAnckVZaCfF6BQCQV1DSUKknWCA4FVrr7anffAlwHnBSz3/8AbgLWJlgWoJAsTvpTRGQgWrM4XZIMBJOBZyPPu8NtRWY2GfgAcEWC5SjS7KMi6aA1i9MlyUAQd8Ut/1//AXCeu/dVPZDZGWa22MwW9/T0DLpAmn1UJB3UfTRd2hM8djcwNfJ8CrCmbJ9O4Lqwb/+uwLvNrNfdb47u5O4LgYUAnZ2dg/7q5N3Jqf+oSNNpPYJ0STIQPATMMrOZwHPAqcBHozu4+8zCYzO7Cri1PAg0kqv7qEgqqGkoXRILBO7ea2bnEPQGagOudPcuMzsrfH275AX6l0k5ApE0UNNQuiRZI8DdbwNuK9sWGwDc/fQkywKFcQRJf4qIDEQji9MlYyOLvdhtTUSaSesRpEmiNYK00VKVIulQqBEsf+5VNdfWYcrOo9lr4riGHzczgaBw56HZR0Wab4eOEQD8851PNrkkw8tZb3sT558wu+HHzVAgCH7q7kOk+fbebRx3fukoXt+0tdlFGVYm7dCRyHEzEwjyYSRQ05BIOuwzaXyziyChzCSL82GNQBUCEZH+MhQIlCMQEYmTmUBQoByBiEh/mQkEyhGIiMTLUCAIfqpGICLSX4YCQSFH0OSCiIikTGYCgeeDn0oWi4j0l51AgHIEIiJxMhMIlCMQEYmXoUCgGoGISJzMBQJli0VE+stMIKDYNNTcYoiIpE1mAoFyBCIi8TIUCJQjEBGJk7lAoKUqRUT6y0wgUK5YRCRe5gKBcgQiIv1lJhAUcwSZOWMRkdpk5rJYSharRiAiEpWhQNDsEoiIpFNmAoGrRiAiEis7gSD8qUAgItJfZgKBBpSJiMTLTiDQwjQiIrGyEwi0VKWISKzMBAINKBMRiZedQKClKkVEYmUmEGgaahGReBkKBMoRiIjEyUwg8GIgUCQQEYnKUCAIfipHICLSX2YCgXIEIiLxMhQIlCMQEYmTuUCgGoGISH+JBgIzO97MnjCzlWZ2fszrHzOzpeG/+83sgKTKUlyqMqkPEBEZphILBGbWBlwGnADMBU4zs7lluz0FvM3d9we+CSxMqjzFZLGyxSIi/SRZIzgUWOnuq919C3AdcFJ0B3e/391fDp8+CExJqjCafVREJF6SgWAy8GzkeXe4rZJPAb+Ne8HMzjCzxWa2uKenZ1CFyWscgYhIrCQDQdwVN3bBSDN7O0EgOC/udXdf6O6d7t45ceLEQRVGk86JiMRrT/DY3cDUyPMpwJryncxsf+AnwAnuvi6pwhRrBEl9gIjIMJVkjeAhYJaZzTSzkcCpwC3RHcxsGvAL4OPu/mSCZVGNQESkgsRqBO7ea2bnAHcAbcCV7t5lZmeFr18BXARMAC4P2+573b0zifJoQJmISLwkm4Zw99uA28q2XRF5/Gng00mWoUBTTIiIxMvMyOLC7KO5zJyxiEhtMnNZzBdHFqtGICISlZlAsPuOHbxnvz0Y35Foa5iIyLCTmaviIdN35pDpOze7GCIiqZOZGoGIiMRTIBARyTgFAhGRjFMgEBHJOAUCEZGMUyAQEck4BQIRkYxTIBARyTgrzMEzXJhZD/C3Qb59V+ClBhZnONA5Z4POORuGcs7T3T12Za9hFwiGwswWJzXNdVrpnLNB55wNSZ2zmoZERDJOgUBEJOOyFggWNrsATaBzzgadczYkcs6ZyhGIiBIHJpsAAAS+SURBVMi2slYjEBGRMgoEIiIZl5lAYGbHm9kTZrbSzM5vdnkaxcyuNLO1ZrY8sm0XM7vLzFaEP3eOvHZB+Dt4wsyOa06ph8bMpprZPWb2mJl1mdkXwu0te95m1mFmi8zs0fCcvx5ub9lzBjCzNjP7i5ndGj5v6fMFMLOnzWyZmS0xs8XhtmTP291b/h/QBqwC9gJGAo8Cc5tdrgad21HAwcDyyLbvAeeHj88Hvhs+nhue+yhgZvg7aWv2OQzinPcADg4fjweeDM+tZc8bMGBc+HgE8Gdgfiufc3ge5wI/BW4Nn7f0+Ybn8jSwa9m2RM87KzWCQ4GV7r7a3bcA1wEnNblMDeHufwDWl20+Cbg6fHw18P7I9uvcfbO7PwWsJPjdDCvu/ry7PxI+fh14DJhMC5+3B94In44I/zktfM5mNgV4D/CTyOaWPd8BJHreWQkEk4FnI8+7w22tapK7Pw/BRRPYLdzecr8HM5sBHERwh9zS5x02kywB1gJ3uXurn/MPgH8E8pFtrXy+BQ7caWYPm9kZ4bZEzzsri9dbzLYs9pttqd+DmY0DbgK+6O6vmcWdXrBrzLZhd97u3gccaGY7Ab80s32r7D6sz9nMTgTWuvvDZnZ0LW+J2TZszrfMEe6+xsx2A+4ys8er7NuQ885KjaAbmBp5PgVY06SybA8vmtkeAOHPteH2lvk9mNkIgiBwrbv/Itzc8ucN4O6vAPcCx9O653wE8D4ze5qgKfcdZvYftO75Frn7mvDnWuCXBE09iZ53VgLBQ8AsM5tpZiOBU4FbmlymJN0CLAgfLwB+Fdl+qpmNMrOZwCxgURPKNyQW3Pr/G/CYu//fyEste95mNjGsCWBmo4F3Ao/Toufs7he4+xR3n0Hw9/o7d//vtOj5FpjZWDMbX3gMvAtYTtLn3ewM+XbMxL+boHfJKuDCZpengef1M+B5YCvB3cGngAnA3cCK8Ocukf0vDH8HTwAnNLv8gzznIwmqv0uBJeG/d7fyeQP7A38Jz3k5cFG4vWXPOXIeR1PqNdTS50vQs/HR8F9X4VqV9HlrigkRkYzLStOQiIhUoEAgIpJxCgQiIhmnQCAiknEKBCIiGadAIFLGzPrCmR8L/xo2W62ZzYjOFCuSBlmZYkKkHhvd/cBmF0Jke1GNQKRG4Tzx3w3XBVhkZnuH26eb2d1mtjT8OS3cPsnMfhmuIfComR0eHqrNzH4critwZzhSWKRpFAhEtjW6rGnolMhrr7n7ocClBLNjEj6+xt33B64FLgm3XwL83t0PIFgzoivcPgu4zN3nAa8AJyd8PiJVaWSxSBkze8Pdx8Vsfxp4h7uvDie9e8HdJ5jZS8Ae7r413P68u+9qZj3AFHffHDnGDIIppGeFz88DRrj7/0n+zETiqUYgUh+v8LjSPnE2Rx73oVydNJkCgUh9Ton8fCB8fD/BDJkAHwP+FD6+G/gsFBeV2WF7FVKkHroTEdnW6HAlsILb3b3QhXSUmf2Z4CbqtHDb54ErzewfgB7gk+H2LwALzexTBHf+nyWYKVYkVZQjEKlRmCPodPeXml0WkUZS05CISMapRiAiknGqEYiIZJwCgYhIxikQiIhknAKBiEjGKRCIiGTcfwEVKozmDXR8zQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 利用鸢尾花数据集，实现前向传播、反向传播，可视化loss曲线\n",
    "\n",
    "# 导入所需模块\n",
    "import tensorflow as tf\n",
    "from sklearn import datasets\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import time  ##1##\n",
    "\n",
    "# 导入数据，分别为输入特征和标签\n",
    "x_data = datasets.load_iris().data\n",
    "y_data = datasets.load_iris().target\n",
    "\n",
    "# 随机打乱数据（因为原始数据是顺序的，顺序不打乱会影响准确率）\n",
    "# seed: 随机数种子，是一个整数，当设置之后，每次生成的随机数都一样（为方便教学，以保每位同学结果一致）\n",
    "np.random.seed(116)  # 使用相同的seed，保证输入特征和标签一一对应\n",
    "np.random.shuffle(x_data)\n",
    "np.random.seed(116)\n",
    "np.random.shuffle(y_data)\n",
    "tf.random.set_seed(116)\n",
    "\n",
    "# 将打乱后的数据集分割为训练集和测试集，训练集为前120行，测试集为后30行\n",
    "x_train = x_data[:-30]\n",
    "y_train = y_data[:-30]\n",
    "x_test = x_data[-30:]\n",
    "y_test = y_data[-30:]\n",
    "\n",
    "# 转换x的数据类型，否则后面矩阵相乘时会因数据类型不一致报错\n",
    "x_train = tf.cast(x_train, tf.float32)\n",
    "x_test = tf.cast(x_test, tf.float32)\n",
    "\n",
    "# from_tensor_slices函数使输入特征和标签值一一对应。（把数据集分批次，每个批次batch组数据）\n",
    "train_db = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(32)\n",
    "test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)\n",
    "\n",
    "# 生成神经网络的参数，4个输入特征故，输入层为4个输入节点；因为3分类，故输出层为3个神经元\n",
    "# 用tf.Variable()标记参数可训练\n",
    "# 使用seed使每次生成的随机数相同（方便教学，使大家结果都一致，在现实使用时不写seed）\n",
    "w1 = tf.Variable(tf.random.truncated_normal([4, 3], stddev=0.1, seed=1))\n",
    "b1 = tf.Variable(tf.random.truncated_normal([3], stddev=0.1, seed=1))\n",
    "\n",
    "lr = 0.1  # 学习率为0.1\n",
    "train_loss_results = []  # 将每轮的loss记录在此列表中，为后续画loss曲线提供数据\n",
    "test_acc = []  # 将每轮的acc记录在此列表中，为后续画acc曲线提供数据\n",
    "epoch = 500  # 循环500轮\n",
    "loss_all = 0  # 每轮分4个step，loss_all记录四个step生成的4个loss的和\n",
    "\n",
    "##########################################################################\n",
    "v_w, v_b = 0, 0\n",
    "beta = 0.9\n",
    "##########################################################################\n",
    "\n",
    "# 训练部分\n",
    "now_time = time.time()  ##2##\n",
    "for epoch in range(epoch):  # 数据集级别的循环，每个epoch循环一次数据集\n",
    "    for step, (x_train, y_train) in enumerate(train_db):  # batch级别的循环 ，每个step循环一个batch\n",
    "        with tf.GradientTape() as tape:  # with结构记录梯度信息\n",
    "            y = tf.matmul(x_train, w1) + b1  # 神经网络乘加运算\n",
    "            y = tf.nn.softmax(y)  # 使输出y符合概率分布（此操作后与独热码同量级，可相减求loss）\n",
    "            y_ = tf.one_hot(y_train, depth=3)  # 将标签值转换为独热码格式，方便计算loss和accuracy\n",
    "            loss = tf.reduce_mean(tf.square(y_ - y))  # 采用均方误差损失函数mse = mean(sum(y-out)^2)\n",
    "            loss_all += loss.numpy()  # 将每个step计算出的loss累加，为后续求loss平均值提供数据，这样计算的loss更准确\n",
    "        # 计算loss对各个参数的梯度\n",
    "        grads = tape.gradient(loss, [w1, b1])\n",
    "\n",
    "        ##########################################################################\n",
    "        # rmsprop\n",
    "        v_w = beta * v_w + (1 - beta) * tf.square(grads[0])\n",
    "        v_b = beta * v_b + (1 - beta) * tf.square(grads[1])\n",
    "        w1.assign_sub(lr * grads[0] / tf.sqrt(v_w))\n",
    "        b1.assign_sub(lr * grads[1] / tf.sqrt(v_b))\n",
    "    ##########################################################################\n",
    "\n",
    "    # 每个epoch，打印loss信息\n",
    "    print(\"Epoch {}, loss: {}\".format(epoch, loss_all / 4))\n",
    "    train_loss_results.append(loss_all / 4)  # 将4个step的loss求平均记录在此变量中\n",
    "    loss_all = 0  # loss_all归零，为记录下一个epoch的loss做准备\n",
    "\n",
    "    # 测试部分\n",
    "    # total_correct为预测对的样本个数, total_number为测试的总样本数，将这两个变量都初始化为0\n",
    "    total_correct, total_number = 0, 0\n",
    "    for x_test, y_test in test_db:\n",
    "        # 使用更新后的参数进行预测\n",
    "        y = tf.matmul(x_test, w1) + b1\n",
    "        y = tf.nn.softmax(y)\n",
    "        pred = tf.argmax(y, axis=1)  # 返回y中最大值的索引，即预测的分类\n",
    "        # 将pred转换为y_test的数据类型\n",
    "        pred = tf.cast(pred, dtype=y_test.dtype)\n",
    "        # 若分类正确，则correct=1，否则为0，将bool型的结果转换为int型\n",
    "        correct = tf.cast(tf.equal(pred, y_test), dtype=tf.int32)\n",
    "        # 将每个batch的correct数加起来\n",
    "        correct = tf.reduce_sum(correct)\n",
    "        # 将所有batch中的correct数加起来\n",
    "        total_correct += int(correct)\n",
    "        # total_number为测试的总样本数，也就是x_test的行数，shape[0]返回变量的行数\n",
    "        total_number += x_test.shape[0]\n",
    "    # 总的准确率等于total_correct/total_number\n",
    "    acc = total_correct / total_number\n",
    "    test_acc.append(acc)\n",
    "    print(\"Test_acc:\", acc)\n",
    "    print(\"--------------------------\")\n",
    "total_time = time.time() - now_time  ##3##\n",
    "print(\"total_time\", total_time)  ##4##\n",
    "\n",
    "# 绘制 loss 曲线\n",
    "plt.title('Loss Function Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Loss')  # y轴变量名称\n",
    "plt.plot(train_loss_results, label=\"$Loss$\")  # 逐点画出trian_loss_results值并连线，连线图标是Loss\n",
    "plt.legend()  # 画出曲线图标\n",
    "plt.show()  # 画出图像\n",
    "\n",
    "# 绘制 Accuracy 曲线\n",
    "plt.title('Acc Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Acc')  # y轴变量名称\n",
    "plt.plot(test_acc, label=\"$Accuracy$\")  # 逐点画出test_acc值并连线，连线图标是Accuracy\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# 请将loss曲线、ACC曲线、total_time记录到 class2\\优化器对比.docx  对比各优化器收敛情况\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adam算法-同时结合SGDM一阶动量和RMSProp二阶动量\n",
    "\n",
    "$$ m_t = \\beta *m_{t-1}+(1-\\beta)g_t $$\n",
    "修正一阶动量的偏差：\n",
    "             $$ \\widehat m_t = \\frac{m_t}{1-\\beta _1^t} $$  \n",
    "               \n",
    "$$ v_t = \\beta*v_{t-1}+(1-\\beta)g_t^2 $$\n",
    "修正二阶动量的偏差：\n",
    "             $$ \\widehat v_t = \\frac{v_t}{1-\\beta _2^t} $$  \n",
    "               \n",
    "$$ \\eta _t = \\frac{lr*\\widehat m_t}{\\sqrt{\\widehat v_t}} ={lr*\\frac{m_t}{1-\\beta _1^t} }\\div{ \\sqrt{\\frac{v_t}{1-\\beta _2^t}}} $$\n",
    "$$ w_{t+1} = w_t - \\eta _t = w_t - {lr*\\frac{m_t}{1-\\beta _1^t} }\\div{ \\sqrt{\\frac{v_t}{1-\\beta _2^t}}} $$\n",
    "\n",
    "        m_w, m_b = 0, 0\n",
    "        v_w, v_b = 0, 0\n",
    "        beta1, beta2 = 0.9, 0.999\n",
    "        delta_w, delta_b = 0, 0\n",
    "        global_step = 0\n",
    "\n",
    "        m_w = beta1 * m_w + (1 - beta1) * grads[0]\n",
    "        m_b = beta1 * m_b + (1 - beta1) * grads[1]\n",
    "        v_w = beta2 * v_w + (1 - beta2) * tf.square(grads[0])\n",
    "        v_b = beta2 * v_b + (1 - beta2) * tf.square(grads[1])\n",
    "\n",
    "        m_w_correction = m_w / (1 - tf.pow(beta1, int(global_step)))\n",
    "        m_b_correction = m_b / (1 - tf.pow(beta1, int(global_step)))\n",
    "        v_w_correction = v_w / (1 - tf.pow(beta2, int(global_step)))\n",
    "        v_b_correction = v_b / (1 - tf.pow(beta2, int(global_step)))\n",
    "\n",
    "        w1.assign_sub(lr * m_w_correction / tf.sqrt(v_w_correction))\n",
    "        b1.assign_sub(lr * m_b_correction / tf.sqrt(v_b_correction))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, loss: 0.21984169632196426\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 1, loss: 0.14480622671544552\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 2, loss: 0.10274342261254787\n",
      "Test_acc: 0.6666666666666666\n",
      "--------------------------\n",
      "Epoch 3, loss: 0.08922167308628559\n",
      "Test_acc: 0.5333333333333333\n",
      "--------------------------\n",
      "Epoch 4, loss: 0.08600791450589895\n",
      "Test_acc: 0.9\n",
      "--------------------------\n",
      "Epoch 5, loss: 0.06994976289570332\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 6, loss: 0.06724478956311941\n",
      "Test_acc: 0.8\n",
      "--------------------------\n",
      "Epoch 7, loss: 0.06104544270783663\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 8, loss: 0.055738057009875774\n",
      "Test_acc: 0.9333333333333333\n",
      "--------------------------\n",
      "Epoch 9, loss: 0.05405292101204395\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 10, loss: 0.04909192584455013\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 11, loss: 0.04825872555375099\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 12, loss: 0.044586071744561195\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 13, loss: 0.04371032351627946\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 14, loss: 0.04151798691600561\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 15, loss: 0.04042437160387635\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 16, loss: 0.039219165686517954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 17, loss: 0.03770230943337083\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 18, loss: 0.037467426154762506\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 19, loss: 0.03545612841844559\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 20, loss: 0.03618234908208251\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 21, loss: 0.03353227535262704\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 22, loss: 0.03510434413328767\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 23, loss: 0.03179901000112295\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 24, loss: 0.03405483299866319\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 25, loss: 0.03026510076597333\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 26, loss: 0.03288380429148674\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 27, loss: 0.028981768060475588\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 28, loss: 0.03153964690864086\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 29, loss: 0.02803809940814972\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 30, loss: 0.030121163930743933\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 31, loss: 0.027460433542728424\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 32, loss: 0.0287771113216877\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 33, loss: 0.027132862713187933\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 34, loss: 0.027615335769951344\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 35, loss: 0.026862118393182755\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 36, loss: 0.02668721415102482\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 37, loss: 0.026502934750169516\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 38, loss: 0.025996240321546793\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 39, loss: 0.026023705024272203\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 40, loss: 0.025497347582131624\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 41, loss: 0.025481836404651403\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 42, loss: 0.02510706940665841\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 43, loss: 0.024957630783319473\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 44, loss: 0.02474378887563944\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 45, loss: 0.02450239099562168\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 46, loss: 0.024368164129555225\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 47, loss: 0.024121190886944532\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 48, loss: 0.023986989865079522\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 49, loss: 0.023787257494404912\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 50, loss: 0.02362554776482284\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 51, loss: 0.023471765918657184\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 52, loss: 0.023299254709854722\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 53, loss: 0.023163855308666825\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 54, loss: 0.023004947928711772\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 55, loss: 0.022868582978844643\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 56, loss: 0.02273069927468896\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 57, loss: 0.022593224188312888\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 58, loss: 0.022468635579571128\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 59, loss: 0.022338303504511714\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 60, loss: 0.02221843833103776\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 61, loss: 0.0220991347450763\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 62, loss: 0.02198242675513029\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 63, loss: 0.02187163825146854\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 64, loss: 0.02176067023538053\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 65, loss: 0.021654900163412094\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 66, loss: 0.021550956647843122\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 67, loss: 0.02144931606017053\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 68, loss: 0.021351238479837775\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 69, loss: 0.021254493622109294\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 70, loss: 0.02116078627295792\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 71, loss: 0.021069139475002885\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 72, loss: 0.020979418884962797\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 73, loss: 0.020892156986519694\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 74, loss: 0.020806539803743362\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 75, loss: 0.02072301576845348\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 76, loss: 0.02064135018736124\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 77, loss: 0.02056133165024221\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 78, loss: 0.02048318414017558\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 79, loss: 0.0204065868165344\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 80, loss: 0.02033161325380206\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 81, loss: 0.02025822177529335\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 82, loss: 0.020186253124848008\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 83, loss: 0.020115777850151062\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 84, loss: 0.02004667604342103\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 85, loss: 0.01997890556231141\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 86, loss: 0.0199124647770077\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 87, loss: 0.019847264979034662\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 88, loss: 0.01978328824043274\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 89, loss: 0.01972051989287138\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 90, loss: 0.01965886657126248\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 91, loss: 0.019598341081291437\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 92, loss: 0.019538887310773134\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 93, loss: 0.01948048430494964\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 94, loss: 0.019423091085627675\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 95, loss: 0.019366701133549213\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 96, loss: 0.01931125670671463\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 97, loss: 0.01925673265941441\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 98, loss: 0.019203137373551726\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 99, loss: 0.01915040146559477\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 100, loss: 0.019098521675914526\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 101, loss: 0.019047497073188424\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 102, loss: 0.018997250590473413\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 103, loss: 0.018947811797261238\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 104, loss: 0.018899145536124706\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 105, loss: 0.018851208966225386\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 106, loss: 0.018804015358909965\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 107, loss: 0.018757533514872193\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 108, loss: 0.018711725482717156\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 109, loss: 0.01866661780513823\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 110, loss: 0.018622154369950294\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 111, loss: 0.018578327959403396\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 112, loss: 0.01853515161201358\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 113, loss: 0.018492562463507056\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 114, loss: 0.01845057145692408\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 115, loss: 0.018409202340990305\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 116, loss: 0.0183683552313596\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 117, loss: 0.018328094854950905\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 118, loss: 0.018288368359208107\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 119, loss: 0.018249179236590862\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 120, loss: 0.018210510723292828\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 121, loss: 0.018172352109104395\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 122, loss: 0.018134684534743428\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 123, loss: 0.018097512423992157\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 124, loss: 0.018060825299471617\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 125, loss: 0.018024582648649812\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 126, loss: 0.017988827312365174\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 127, loss: 0.01795349083840847\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 128, loss: 0.017918606521561742\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 129, loss: 0.0178841648157686\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 130, loss: 0.01785011379979551\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 131, loss: 0.017816497944295406\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 132, loss: 0.01778329908847809\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 133, loss: 0.017750458559021354\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 134, loss: 0.01771804573945701\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 135, loss: 0.01768598728813231\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 136, loss: 0.017654324183240533\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 137, loss: 0.017623017309233546\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 138, loss: 0.01759206922724843\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 139, loss: 0.017561486922204494\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 140, loss: 0.01753123663365841\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 141, loss: 0.01750132767483592\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 142, loss: 0.017471773084253073\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 143, loss: 0.017442532116547227\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 144, loss: 0.017413603840395808\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 145, loss: 0.01738501782529056\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 146, loss: 0.01735673495568335\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 147, loss: 0.01732874009758234\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 148, loss: 0.017301083775237203\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 149, loss: 0.01727369101718068\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 150, loss: 0.01724658557213843\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 151, loss: 0.017219797242432833\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 152, loss: 0.017193264327943325\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 153, loss: 0.017167016165331006\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 154, loss: 0.01714104856364429\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 155, loss: 0.017115335213020444\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 156, loss: 0.01708989217877388\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 157, loss: 0.01706470688804984\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 158, loss: 0.017039780505001545\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 159, loss: 0.017015096498653293\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 160, loss: 0.016990668140351772\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 161, loss: 0.01696647983044386\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 162, loss: 0.016942527145147324\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 163, loss: 0.01691881287842989\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 164, loss: 0.016895329114049673\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 165, loss: 0.016872075852006674\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 166, loss: 0.01684904331341386\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 167, loss: 0.016826246632263064\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 168, loss: 0.016803656239062548\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 169, loss: 0.01678128121420741\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 170, loss: 0.016759122721850872\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 171, loss: 0.016737161902710795\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 172, loss: 0.01671542041003704\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 173, loss: 0.016693870769813657\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 174, loss: 0.016672536032274365\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 175, loss: 0.016651404788717628\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 176, loss: 0.016630435828119516\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 177, loss: 0.01660967990756035\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 178, loss: 0.01658910745754838\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 179, loss: 0.01656872034072876\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 180, loss: 0.016548524843528867\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 181, loss: 0.016528500244021416\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 182, loss: 0.01650867285206914\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 183, loss: 0.01648899936117232\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 184, loss: 0.016469511203467846\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 185, loss: 0.01645019114948809\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 186, loss: 0.016431048745289445\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 187, loss: 0.016412061406299472\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 188, loss: 0.01639324170537293\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 189, loss: 0.016374601516872644\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 190, loss: 0.016356092877686024\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 191, loss: 0.016337770270183682\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 192, loss: 0.0163195903878659\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 193, loss: 0.016301556956022978\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 194, loss: 0.016283699311316013\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 195, loss: 0.016265966929495335\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 196, loss: 0.0162483979947865\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 197, loss: 0.01623098086565733\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 198, loss: 0.01621368434280157\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 199, loss: 0.01619656104594469\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 200, loss: 0.016179556725546718\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 201, loss: 0.016162703512236476\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 202, loss: 0.016145988134667277\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 203, loss: 0.016129403607919812\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 204, loss: 0.016112956684082747\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 205, loss: 0.016096646199002862\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 206, loss: 0.016080460976809263\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 207, loss: 0.016064407536759973\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 208, loss: 0.016048484481871128\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 209, loss: 0.016032679239287972\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 210, loss: 0.016017013229429722\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 211, loss: 0.016001457814127207\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 212, loss: 0.015986030688509345\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 213, loss: 0.015970737673342228\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 214, loss: 0.015955530107021332\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 215, loss: 0.015940470853820443\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 216, loss: 0.015925521729514003\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 217, loss: 0.015910674817860126\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 218, loss: 0.015895960154011846\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 219, loss: 0.01588133769109845\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 220, loss: 0.015866847708821297\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 221, loss: 0.015852457843720913\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 222, loss: 0.01583817577920854\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 223, loss: 0.01582400780171156\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 224, loss: 0.015809929696843028\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 225, loss: 0.015795978717505932\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 226, loss: 0.01578212040476501\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 227, loss: 0.015768363839015365\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 228, loss: 0.015754722524434328\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 229, loss: 0.015741158509626985\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 230, loss: 0.015727709978818893\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 231, loss: 0.01571436133235693\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 232, loss: 0.015701100463047624\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 233, loss: 0.0156879466958344\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 234, loss: 0.015674876514822245\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 235, loss: 0.01566191460005939\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 236, loss: 0.015649035573005676\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 237, loss: 0.015636248979717493\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 238, loss: 0.015623555053025484\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 239, loss: 0.015610962640494108\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 240, loss: 0.015598438447341323\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 241, loss: 0.01558601763099432\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 242, loss: 0.015573689015582204\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 243, loss: 0.01556142303161323\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 244, loss: 0.015549274859949946\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 245, loss: 0.01553719921503216\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 246, loss: 0.015525190392509103\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 247, loss: 0.015513295191340148\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 248, loss: 0.015501460409723222\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 249, loss: 0.015489712823182344\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 250, loss: 0.01547804637812078\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 251, loss: 0.015466455952264369\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 252, loss: 0.01545494666788727\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 253, loss: 0.015443526324816048\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 254, loss: 0.015432174550369382\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 255, loss: 0.015420897048898041\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 256, loss: 0.015409702202305198\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 257, loss: 0.015398564864881337\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 258, loss: 0.015387521940283477\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 259, loss: 0.015376550145447254\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 260, loss: 0.01536564074922353\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 261, loss: 0.015354805742390454\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 262, loss: 0.015344054787419736\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 263, loss: 0.015333368093706667\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 264, loss: 0.015322739374823868\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 265, loss: 0.015312209376133978\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 266, loss: 0.015301715466193855\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 267, loss: 0.01529130304697901\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 268, loss: 0.015280971536412835\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 269, loss: 0.015270684263668954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 270, loss: 0.015260477201081812\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 271, loss: 0.015250333934091032\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 272, loss: 0.015240264474414289\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 273, loss: 0.015230227611027658\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 274, loss: 0.015220290166325867\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 275, loss: 0.015210395096801221\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 276, loss: 0.015200563124381006\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 277, loss: 0.015190812177024782\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 278, loss: 0.015181106980890036\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 279, loss: 0.015171458944678307\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 280, loss: 0.01516188238747418\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 281, loss: 0.015152363339439034\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 282, loss: 0.015142893185839057\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 283, loss: 0.015133499749936163\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 284, loss: 0.015124144847504795\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 285, loss: 0.01511485071387142\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 286, loss: 0.015105649014003575\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 287, loss: 0.01509645115584135\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 288, loss: 0.015087344567291439\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 289, loss: 0.01507828023750335\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 290, loss: 0.015069269924424589\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 291, loss: 0.015060317236930132\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 292, loss: 0.015051418798975646\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 293, loss: 0.015042576589621603\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 294, loss: 0.015033785603009164\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 295, loss: 0.015025043627247214\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 296, loss: 0.01501635240856558\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 297, loss: 0.01500773336738348\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 298, loss: 0.014999138074927032\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 299, loss: 0.014990606694482267\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 300, loss: 0.014982131193391979\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 301, loss: 0.014973694807849824\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 302, loss: 0.014965313719585538\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 303, loss: 0.014956979546695948\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 304, loss: 0.014948692522011697\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 305, loss: 0.014940451714210212\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 306, loss: 0.014932271442376077\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 307, loss: 0.014924131100997329\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 308, loss: 0.01491604174952954\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 309, loss: 0.014907981851138175\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 310, loss: 0.014899994130246341\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 311, loss: 0.014892029343172908\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 312, loss: 0.01488412159960717\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 313, loss: 0.014876257511787117\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 314, loss: 0.014868434402160347\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 315, loss: 0.014860664261505008\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 316, loss: 0.01485293300356716\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 317, loss: 0.014845237950794399\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 318, loss: 0.014837599825114012\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 319, loss: 0.014829996740445495\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 320, loss: 0.01482243521604687\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 321, loss: 0.014814923633821309\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 322, loss: 0.014807445695623755\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 323, loss: 0.014800010365433991\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 324, loss: 0.014792613917961717\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 325, loss: 0.014785272884182632\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 326, loss: 0.014777956530451775\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 327, loss: 0.014770680107176304\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 328, loss: 0.014763458399102092\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 329, loss: 0.014756269636563957\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 330, loss: 0.0147491100942716\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 331, loss: 0.014742006896995008\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 332, loss: 0.014734933385625482\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 333, loss: 0.014727887813933194\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 334, loss: 0.014720903942361474\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 335, loss: 0.014713941724039614\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 336, loss: 0.01470699580386281\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 337, loss: 0.014700142433866858\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 338, loss: 0.014693279401399195\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 339, loss: 0.014686463284306228\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 340, loss: 0.014679685118608177\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 341, loss: 0.014672953402623534\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 342, loss: 0.014666244969703257\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 343, loss: 0.014659570879302919\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 344, loss: 0.014652951271273196\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 345, loss: 0.01464633469004184\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 346, loss: 0.014639781438745558\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 347, loss: 0.014633247861638665\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 348, loss: 0.014626748627051711\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 349, loss: 0.014620288624428213\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 350, loss: 0.01461386273149401\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 351, loss: 0.014607467572204769\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 352, loss: 0.014601109200157225\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 353, loss: 0.014594775391742587\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 354, loss: 0.014588486403226852\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 355, loss: 0.014582205447368324\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 356, loss: 0.014575987355783582\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 357, loss: 0.014569777180440724\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 358, loss: 0.014563610311597586\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 359, loss: 0.0145574671914801\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 360, loss: 0.014551367261447012\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 361, loss: 0.014545284095220268\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 362, loss: 0.014539248310029507\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 363, loss: 0.014533231500536203\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 364, loss: 0.014527247403748333\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 365, loss: 0.014521289966069162\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 366, loss: 0.014515359071083367\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 367, loss: 0.014509470434859395\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 368, loss: 0.01450359751470387\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 369, loss: 0.01449776126537472\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 370, loss: 0.01449196261819452\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 371, loss: 0.014486179919913411\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 372, loss: 0.014480430050753057\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 373, loss: 0.014474703697487712\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 374, loss: 0.01446901133749634\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 375, loss: 0.014463349594734609\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 376, loss: 0.01445770321879536\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 377, loss: 0.014452091767452657\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 378, loss: 0.014446514542214572\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 379, loss: 0.014440948492847383\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 380, loss: 0.014435417484492064\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 381, loss: 0.014429923496209085\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 382, loss: 0.014424436609260738\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 383, loss: 0.014418995124287903\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 384, loss: 0.014413560158573091\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 385, loss: 0.014408164541237056\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 386, loss: 0.014402791042812169\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 387, loss: 0.01439743919763714\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 388, loss: 0.014392129145562649\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 389, loss: 0.014386819442734122\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 390, loss: 0.014381557703018188\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 391, loss: 0.0143763090018183\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 392, loss: 0.014371079741977155\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 393, loss: 0.014365882729180157\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 394, loss: 0.014360724366270006\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 395, loss: 0.014355559251271188\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 396, loss: 0.014350439189001918\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 397, loss: 0.014345344854518771\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 398, loss: 0.01434026483912021\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 399, loss: 0.014335198211483657\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 400, loss: 0.014330179779790342\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 401, loss: 0.014325177879072726\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 402, loss: 0.014320179470814764\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 403, loss: 0.01431523624341935\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 404, loss: 0.014310296275652945\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 405, loss: 0.014305356540717185\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 406, loss: 0.014300484559498727\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 407, loss: 0.014295612811110914\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 408, loss: 0.014290758990682662\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 409, loss: 0.014285923563875258\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 410, loss: 0.014281123992986977\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 411, loss: 0.014276350964792073\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 412, loss: 0.014271582127548754\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 413, loss: 0.014266838901676238\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 414, loss: 0.01426211791113019\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 415, loss: 0.014257428119890392\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 416, loss: 0.014252756023779511\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 417, loss: 0.014248086954466999\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 418, loss: 0.014243459678255022\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 419, loss: 0.01423885056283325\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 420, loss: 0.014234248432330787\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 421, loss: 0.014229674939997494\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 422, loss: 0.01422511984128505\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 423, loss: 0.014220590004697442\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 424, loss: 0.014216066687367857\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 425, loss: 0.014211585163138807\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 426, loss: 0.01420710829552263\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 427, loss: 0.014202650636434555\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 428, loss: 0.014198216726072133\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 429, loss: 0.014193804701790214\n",
      "Test_acc: 1.0\n",
      "--------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 430, loss: 0.01418939686845988\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 431, loss: 0.014185020816512406\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 432, loss: 0.014180669910274446\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 433, loss: 0.01417631993535906\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 434, loss: 0.014172004302963614\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 435, loss: 0.014167713117785752\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 436, loss: 0.014163423562422395\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 437, loss: 0.014159139012917876\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 438, loss: 0.014154912671074271\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 439, loss: 0.014150677714496851\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 440, loss: 0.014146466157399118\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 441, loss: 0.014142281608656049\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 442, loss: 0.014138103229925036\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 443, loss: 0.014133928925730288\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 444, loss: 0.014129797695204616\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 445, loss: 0.014125677989795804\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 446, loss: 0.014121563988737762\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 447, loss: 0.014117479673586786\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 448, loss: 0.014113407698459923\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 449, loss: 0.014109352021478117\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 450, loss: 0.014105312759056687\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 451, loss: 0.01410128886345774\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 452, loss: 0.014097290462814271\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 453, loss: 0.014093308127485216\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 454, loss: 0.014089317643083632\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 455, loss: 0.014085388625971973\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 456, loss: 0.01408143702428788\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 457, loss: 0.014077513595111668\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 458, loss: 0.014073620433919132\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 459, loss: 0.014069717843085527\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 460, loss: 0.014065848314203322\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 461, loss: 0.014061982743442059\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 462, loss: 0.014058151515200734\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 463, loss: 0.014054323663003743\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 464, loss: 0.014050516998395324\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 465, loss: 0.01404671196360141\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 466, loss: 0.014042934984900057\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 467, loss: 0.014039175701327622\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 468, loss: 0.01403541408944875\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 469, loss: 0.014031683676876128\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 470, loss: 0.014027963043190539\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 471, loss: 0.014024264295585454\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 472, loss: 0.01402056182269007\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 473, loss: 0.01401689334306866\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 474, loss: 0.014013235457241535\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 475, loss: 0.014009579899720848\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 476, loss: 0.014005955075845122\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 477, loss: 0.014002340962179005\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 478, loss: 0.013998735812492669\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 479, loss: 0.013995141140185297\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 480, loss: 0.013991574640385807\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 481, loss: 0.013988003483973444\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 482, loss: 0.013984457938931882\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 483, loss: 0.013980936608277261\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 484, loss: 0.013977404916658998\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 485, loss: 0.01397391117643565\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 486, loss: 0.013970413361676037\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 487, loss: 0.01396693557035178\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 488, loss: 0.013963473029434681\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 489, loss: 0.013960016192868352\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 490, loss: 0.013956574373878539\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 491, loss: 0.013953150599263608\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 492, loss: 0.013949743937700987\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 493, loss: 0.013946344261057675\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 494, loss: 0.013942962163127959\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 495, loss: 0.013939584256149828\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 496, loss: 0.013936232659034431\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 497, loss: 0.013932874542661011\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 498, loss: 0.013929549255408347\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "Epoch 499, loss: 0.013926226296462119\n",
      "Test_acc: 1.0\n",
      "--------------------------\n",
      "total_time 14.59939432144165\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dfZRddX3v8ffnnJnJzCSThCQTCAkQwIiGBwVSLlzRilIL9gF66apYW61LFosWr3WJVrz1eqvXdkmftLRaRE1FEenDFUQbEcWqxWJl0EDCk4QQmskDmYQ8Zx7PfO8fe5+ZPTN7wpkwJ2dyzue11lln79/e+5zfb1ZyPue3f/v8tiICMzOz8Qq1roCZmc1MDggzM8vlgDAzs1wOCDMzy+WAMDOzXA4IMzPL5YAwqwFJBySdVut6mB2OA8JqRtImSZfW4H2/KGkg/ZAuP95Sxff7vqRrsmURMSciNlbp/X5bUlfarm2SviXp4mq8l9U3B4Q1qj9PP6TLj3+sdYWmg6T3AZ8C/gw4HjgZ+AxwxRG8VtP01s6ONQ4Im3EkzZL0KUlb08enJM1Kty2S9E1JeyS9IOnfJRXSbR+UtEXSfklPSXrjFN/3i5I+nll/vaTuzPomSe+X9KikvZL+UVJrZvsVktZK2ifpGUmXSfpT4LXA36Xf6P8u3TckvSxdnifpS5J6JD0n6cOZNv2epAck/aWk3ZKelXT5JPWfB3wMuD4ivhYRByNiMCK+EREfmEIbPyjpUeBgWpd/Gfc+fyPp5kzdv5D2VLZI+rik4lT+7jZz+RuCzUR/DFwIvBoI4OvAh4H/DdwAdAOd6b4XAiHpDODdwC9ExFZJy4FqfFD9FnAZ0Af8CPg94BZJFwBfAn4TuB9YAnRExL2SXgPcHhGfn+Q1/xaYB5wGLATuA7YBX0i3/zfgNmARcC3wBUlLY+I8ORcBrcBdL7GNbwV+BdgJLAb+l6S5EbEv/fD/LeA30n1vA54HXgbMBr4JbAY++xLrYDOAexA2E70N+FhE7IiIHuCjwO+m2wZJPnxPSb8d/3v6QVkCZgErJTVHxKaIeOYw7/H+tBeyR9LOKdTt5ojYGhEvAN8gCTGAdwGrI+I7ETEcEVsi4skXe7H0A/ctwIciYn9EbAL+KtNegOci4nMRUSL5QF5CcvpovIXAzogYmkJ78twcEZsjojcingN+ClyZbnsDcCgifizpeOBy4L1pb2UH8Eng6pf4/jZDOCBsJjoReC6z/lxaBvAXwAbgPkkbJd0IEBEbgPcCfwLskHSnpBOZ3F9GxPz0sWgKddueWT4EzEmXTwIOF0iTWQS0MLG9S/PeMyIOpYtzmGgXsGgaxg42j1u/g6RXAfDb6TrAKUAzsK0ctiQ9h8Uv8f1thnBA2Ey0leTDp+zktIz0W/YNEXEa8GvA+8pjDRFxR0RcnB4bwE1TfN+DQHtm/YQpHLsZOH2SbYebMnknSa9ofHu3TOG9yx4kOfV15WH2qaSN4+v7z8DrJS0jObVUDojNQD+wKBO2cyPizCOou81ADgirtWZJrZlHE/BV4MOSOiUtAj4C3A4g6VclvUySgH0kp5ZKks6Q9IZ0MLsP6E23TcVa4M2SFkg6gaRHUqkvAO+U9EZJBUlLJb0i3fY8yfjCBOlpo38C/lRSh6RTgPeV2zsVEbGX5G/1aUlXSmqX1Czpckl/fqRtTE/zfR/4B+DZiHgiLd9GMl7yV5Lmpu0+XdIvTrXuNjM5IKzW1pB8mJcffwJ8HOgCHgXWkZwDL195swL4LnCA5BvzZyLi+yTjD58g+Ua+nXRwdYp1+TLwCLCJ5IOv4ktfI+InwDtJzsHvBX7AaK/gb4DfTK9Cujnn8P9J8s1+I/AAyTf01VOse7kef00SMB8Geki+5b8buDvd5UjbeAdwKaO9h7K3k5wiexzYDfwLyRiJ1QH5hkFmZpbHPQgzM8vlgDAzs1wOCDMzy+WAMDOzXHU11caiRYti+fLlta6Gmdkx4+GHH94ZEZ152+oqIJYvX05XV1etq2FmdsyQ9Nxk23yKyczMcjkgzMwslwPCzMxy1dUYhJnZVAwODtLd3U1fX1+tq1J1ra2tLFu2jObm5oqPcUCYWcPq7u6mo6OD5cuXk8z/WJ8igl27dtHd3c2pp55a8XE+xWRmDauvr4+FCxfWdTgASGLhwoVT7ik5IMysodV7OJQdSTsdEMDN9z/ND37eU+tqmJnNKA4I4O+//ww/2jCV2xKbmdU/BwRQEAwP+74YZmZZDgigIOF8MLNa+exnP8v1119f62pM4IAAJBj2nfXMrEYeffRRzj777FpXYwIHBFAoCN961cxqZd26dRMC4sknn+R1r3sdZ555Jpdeeik7dybjpLfddhvnn38+55xzDq997WsnLZsO/qEcySmmkgPCrKF99BuP8fjWfdP6mitPnMv/+bUzX3S/9evXc9ZZZ42s9/f3c9VVV3H77bdz7rnnctNNN/HJT36SG2+8kZtuuom1a9fS0tLCnj172L9//4Sy6eIeBB6DMLPa2bx5Mx0dHcybN2+k7O677+biiy/m3HPPBWDlypXs2LGDYrFIb28vN9xwA11dXcyfPz+3bLq4B0FyFZNPMZk1tkq+6VdD3vjD448/PqZs3bp1rFy5kvb2dtavX883vvENrr32Wq655hr+4A/+ILdsOjggSHsQw7WuhZk1orzxh6VLl7J27VoANm7cyJe//GUeeOABnn76aVasWMHVV1/N448/Tl9fX27ZdHFAkP4Owj0IM6uBdevWce+99/LVr34VgCVLlvC9732PNWvWcPbZZ9PW1sbq1atZuHAhN9xwAw8++CCzZ8/mzDPP5HOf+xzXXXfdhLLp4oAgmaPEYxBmVgtf+cpXcsvvvvvuCWVf/OIXKyqbLh6kBgoF9yDMzMZzQABFyQFhZjaOAwJf5mrWyBrlCsYjaacDAk+1YdaoWltb2bVrV92HRPmOcq2trVM6zoPUJD2Iev8HYmYTLVu2jO7ubnp66v9+MOV7Uk9FVQNC0mXA3wBF4PMR8Ylx298GfDBdPQD8fkQ8Usmx06kgUfI5JrOG09zcPKV7NDeaqp1iklQEPg1cDqwE3ipp5bjdngV+MSLOAf4vcOsUjp3GuuIxCDOzcao5BnEBsCEiNkbEAHAncEV2h4j4j4jYna7+GFhW6bHTqejZXM3MJqhmQCwFNmfWu9OyybwL+NZUj5V0raQuSV1Heh7RVzGZmU1UzYBQTlnux7CkS0gCojweUfGxEXFrRKyKiFWdnZ1HVFFPtWFmNlE1B6m7gZMy68uAreN3knQO8Hng8ojYNZVjp4un2jAzm6iaPYiHgBWSTpXUAlwN3JPdQdLJwNeA342In0/l2OlUEAw7IczMxqhaDyIihiS9G/g2yaWqqyPiMUnXpdtvAT4CLAQ+IwlgKD1dlHtstepaLHiqDTOz8ar6O4iIWAOsGVd2S2b5GuCaSo+tFnkuJjOzCTzVBuVB6lrXwsxsZnFA4Kk2zMzyOCDw7yDMzPI4IEim2vBcTGZmYzkg8FQbZmZ5HBD4FJOZWR4HBJ5qw8wsjwMCT7VhZpbHAUHSg/AYhJnZWA4IfEc5M7M8Dgig4LmYzMwmcEBQ/iV1rWthZjazOCDwVUxmZnkcEPh3EGZmeRwQJFNtuAdhZjaWA4K0B+EuhJnZGA4IoOhTTGZmEzgggELBp5jMzMZzQOCpNszM8jgg8FQbZmZ5HBCUL3N1QJiZZTkg8FxMZmZ5HBB4qg0zszwOCDzVhplZHgcE5dlca10LM7OZxQGBp9owM8vjgMBXMZmZ5XFAUB6DqHUtzMxmFgcE5bmYnBBmZlkOCJKpNiL8a2ozsywHBMkYBODfQpiZZTggSMYgwFcymZllOSBIfgcBUHJAmJmNcECQ/A4CfIrJzCzLAUFyFRP4FJOZWZYDgtFBav8WwsxslAOC0VNM7kGYmY1yQJC5zHW4xhUxM5tBHBCMXubqq5jMzEZVNSAkXSbpKUkbJN2Ys/0Vkh6U1C/p/eO2bZK0TtJaSV3VrGex4EFqM7Pxmqr1wpKKwKeBXwK6gYck3RMRj2d2ewF4D3DlJC9zSUTsrFYdy+SrmMzMJqhmD+ICYENEbIyIAeBO4IrsDhGxIyIeAgarWI8X5ak2zMwmqmZALAU2Z9a707JKBXCfpIclXTvZTpKuldQlqaunp+eIKuqpNszMJqpmQCinbCqfwK+JiPOAy4HrJb0ub6eIuDUiVkXEqs7OziOpp38HYWaWo5oB0Q2clFlfBmyt9OCI2Jo+7wDuIjllVRUjv4NwQpiZjahmQDwErJB0qqQW4GrgnkoOlDRbUkd5GXgTsL5aFfVVTGZmE1XtKqaIGJL0buDbQBFYHRGPSbou3X6LpBOALmAuMCzpvcBKYBFwV3p1URNwR0TcW626+hSTmdlEVQsIgIhYA6wZV3ZLZnk7yamn8fYBr6pm3bLKp5hKTggzsxH+JTXQVEj+DD7FZGY2ygHB6BjEUMkBYWZW5oAAmsp3lPMpJjOzEQ4IoFhMexDDns7VzKzMAYF7EGZmeRwQjN5ydMgBYWY2wgHB6CC1exBmZqMcEEBT0QFhZjaeAwIopr+DcECYmY1yQDA6SO0xCDOzUQ4IsmMQvszVzKzMAYF7EGZmeRwQQMFXMZmZTeCAINOD8FxMZmYjHBBkxiA8m6uZ2QgHBKPTffsUk5nZKAcEmem+HRBmZiMcEGQm6yv5Mlczs7KKAkLSbEmFdPnlkn5dUnN1q3b0jE737R6EmVlZpT2IHwKtkpYC9wPvBL5YrUodbeXZXD0GYWY2qtKAUEQcAv4H8LcR8RvAyupV6+jyGISZ2UQVB4Ski4C3Af+aljVVp0pHX3kMYtgBYWY2otKAeC/wIeCuiHhM0mnAv1WvWkeXexBmZhNV1AuIiB8APwBIB6t3RsR7qlmxo0kSxYI8BmFmllHpVUx3SJoraTbwOPCUpA9Ut2pHV7Eg9yDMzDIqPcW0MiL2AVcCa4CTgd+tWq1qoKkgT/dtZpZRaUA0p797uBL4ekQMAnX1dds9CDOzsSoNiM8Cm4DZwA8lnQLsq1alasFjEGZmY1U6SH0zcHOm6DlJl1SnSrXR5B6EmdkYlQ5Sz5P015K60sdfkfQm6kaxIP8Owswso9JTTKuB/cBvpY99wD9Uq1K10FQouAdhZpZR6a+hT4+IqzLrH5W0thoVqhWPQZiZjVVpD6JX0sXlFUmvAXqrU6Xa8BiEmdlYlfYgrgO+JGleur4beEd1qlQbRf8OwsxsjEqvYnoEeJWkuen6PknvBR6tZuWOpmJBDJXcgzAzK5vSHeUiYl/6i2qA91WhPjXjMQgzs7Feyi1HNW21mAGaCqIUDggzs7KXEhB19WnqHoSZ2ViHHYOQtJ/8IBDQVpUa1UhToeAxCDOzjMP2ICKiIyLm5jw6IuJFB7glXSbpKUkbJN2Ys/0Vkh6U1C/p/VM5drolk/X5KiYzs7KXcorpsCQVgU8Dl5Pcv/qtksbfx/oF4D3AXx7BsdOqraVI72Cpmm9hZnZMqVpAABcAGyJiY0QMAHcCV2R3iIgdEfEQMDjVY6dbe0uRQ/0OCDOzsmoGxFJgc2a9Oy2b1mMlXVueRLCnp+eIKgowZ1YTBweGjvh4M7N6U82AyLsMttJR4IqPjYhbI2JVRKzq7OysuHLjtbc0uQdhZpZRzYDoBk7KrC8Dth6FY4/I7FlFDg4MEf4thJkZUN2AeAhYIelUSS3A1cA9R+HYI9Le0sRwQP+Qr2QyM4PKJ+ubsogYkvRu4NtAEVgdEY9Jui7dfoukE4AuYC4wnM7vtDKd62nCsdWqKyQ9CIAD/UO0Nher+VZmZseEqgUEQESsAdaMK7sls7yd5PRRRcdWU3tL8qc41F+COUfrXc3MZq5qnmI6psxuSXoNvpLJzCzhgEi1z0p7EA4IMzPAATFipAfhS13NzAAHxIiRMQj3IMzMAAfEiI7WJCD29o6f9cPMrDE5IFJL5rXS2lzgqe0Hal0VM7MZwQGRaioWeOWSuazfsrfWVTEzmxEcEBlnnTiPx7fte/EdzcwagAMiY9GcWRzoH2Ko5Ok2zMwcEBnt6aWufZ6PyczMAZHVmgZE74B/C2Fm5oDIaEsn6evzrUfNzBwQWeWA8L2pzcwcEGO0tSR/jkM+xWRm5oDIKt8HwmMQZmYOiDHK8zF5DMLMzAExhscgzMxGOSAy2nyKycxshAMiozUdpHYPwszMATGGexBmZqMcEBkegzAzG+WAyGgqFmgpFhwQZmY4ICZobS74FJOZGQ6ICTo7ZrH5hUO1roaZWc05IMZZdcoCup7bzfBw1LoqZmY15YAY5/zlx7G3d5CNO31vajNrbA6IcU7vnAPAf/k0k5k1OAfEOCfObwVg296+GtfEzKy2HBDjdM6ZRUGw3QFhZg3OATFOU7HA4o5W9yDMrOE5IHIsmd/qHoSZNTwHRI4l81rZure31tUwM6spB0SOE+a2sX1vHxH+LYSZNS4HRI4l81o5NFBiX99QratiZlYzDogcS9JLXT0OYWaNzAGRY8m8JCA8DmFmjcwBkeOEeW0AnrTPzBqaAyLHifNaWb6wnTXrttW6KmZmNeOAyCGJX3/1Un688QUO9Hug2swaU1UDQtJlkp6StEHSjTnbJenmdPujks7LbNskaZ2ktZK6qlnPPC8/Ppm0z6eZzKxRVS0gJBWBTwOXAyuBt0paOW63y4EV6eNa4O/Hbb8kIl4dEauqVc/JnHRcO+CAMLPGVc0exAXAhojYGBEDwJ3AFeP2uQL4UiR+DMyXtKSKdarYSQuSgOje7SuZzKwxVTMglgKbM+vdaVml+wRwn6SHJV072ZtIulZSl6Sunp6eaah24rj2Ztpbir4vhJk1rGoGhHLKxs9dcbh9XhMR55Gchrpe0uvy3iQibo2IVRGxqrOz88hrO75iEquWL+Cfuzb7B3Nm1pCqGRDdwEmZ9WXA1kr3iYjy8w7gLpJTVkfVH/3yGRwcKPHjjbuO9lubmdVcNQPiIWCFpFMltQBXA/eM2+ce4O3p1UwXAnsjYpuk2ZI6ACTNBt4ErK9iXXOdcUIHzUXx1PP7j/Zbm5nVXFO1XjgihiS9G/g2UARWR8Rjkq5Lt98CrAHeDGwADgHvTA8/HrhLUrmOd0TEvdWq62SaiwVO75zDz7c7IMys8VQtIAAiYg1JCGTLbsksB3B9znEbgVdVs26VWnniXO577Hm27Oll6fy2WlfHzOyo8S+pX8QfvnEFA0PD/MMDz9a6KmZmR5UD4kWcsnA2F56+kO89taPWVTEzO6ocEBW49JWL2dhzkIc2vVDrqpiZHTUOiAr85vnLWNwxiz9b84RvQ2pmDcMBUYH2liZueNPL+dl/7eH+J3yqycwagwOiQledt4wT5rbyiXufZOeB/lpXx8ys6hwQFWoqFvjAL5/Bhh0H+MjX1/s+EWZW9xwQU3DV+ct489knsGbddn5v9U/Y1zdY6yqZmVWNA2KK3n7RcgC6ntvNOX9yH3sPOSTMrD45IKbowtMW8pm3jdz4jld97D6e9lxNZlaHHBBH4LUrFrFwdsvI+i998ofcu35bDWtkZjb9HBBHoKO1mQc/9MYxZdfd/lO++/jzNaqRmdn0c0AcoZamAl94x9hbZV/zpS5+7tNNZlYnHBAvwRtfeTxvv+iUMWVv+uQPGRgarlGNzMymjwPiJfrYFWfxjnEh8fIPf4td/jGdmR3jHBDT4MO/upJbfue8MWXnf/y7fGvdNs/dZGbHLNXTB9iqVauiq6urZu/fO1Dib7/3NN9av51ndx4EYNlxbbz9olN4yy+czLy25prVzcwsj6SHI2JV7jYHRHXs7xvkW+u3c8v3n2HjzoM0F8XlZy3hN85bykWnLaS1uVjrKpqZOSBqqTQc/Oezu/jaT7fwr49uo3ewxJxZTVzyisW89mWLuOj0hZy0oL3W1TSzBuWAmCH6Bks8uHEX316/ne8+8Tw7DwwAsHR+G6uWH8eq5Qs4Z+k8zjihwz0MMzsqDhcQTUe7Mo2stbnIJWcs5pIzFhMRPL3jAA8+s4ufPPsCP9qwi6+v3QpAsSBWLJ7D2Uvn8colc3nZ4jmcvngOS+a2Uiioxq0ws0bhHsQMERF07+5l/Za9rN+6l/Vb9rF+y152HRwY2aetuchpnbM5vXMOJy9oZ+lxbSw7ro2l89s4cX6bex1mNmXuQRwDJHHSgnZOWtDO5WcvAZLQ6DnQz8aegzzTc4Bndhxk484D/Gzzbv513TZKw2PDvbNjVhoWrXTOmUVnR/JY3NE6srxwdgtNRV/dbGYvzgExg0licUcriztaufC0hWO2DZWGeX5/P1t299K9+xBbdveyZU8v3bt7+fnzB/jRhl3s7Z04FbkEC9pbmN/ezHHtLcxvb+G49mbmtzenyy3pcvPIckdrM7Nbikg+vWXWSBwQx6imYoGl85PTSxecuiB3n77BEjsP9NOzv58d+5Pnnv399BzoZ/fBAfYcGqR79yHWbxlk96EB+g8zRYgEc2Y10TGriTmtTXS0NjOnvDyrKdnW2jyy3j6rSHtLkbbmJtpaysvFkeXWpqLHU8xmOAdEHWttLrLsuHaWHVfZZbR9gyV2Hxpg98FB9vQmAbL70AAH+oY40D/E/vS5vL7n0ACbdx8aWT80UJpi/Qq0tzSNCY4xIdJcZFZTkVlNBWY1F0aXRx7FtLwwyX5FWsr7puVFh5JZxRwQNqK1uciSeW0smdd2RMcPlYY5OFBif98ghwZKHBoo0TtQondwaGS9b7A0bnloQvmuAwN0DybHDpSG6R8s0T80fNgeTqWaCqK5WKCpKFqKhQnLzU2iqVBI1pvSfQsFWnKWD/s66XpzURQLBZoKolhQ5rmQPBcnKS+vFycpT5992s+qyQFh06apWGBeW6FqU4pERBIYQ8P0Dw7TP1QaWR4fJP1DpXSfsfv1D5UYGg4GhoYZLA0zVAoGS8nx2eXBUrL/gb4hBkrBUFo2WIp039HlwdIwtboYsDgheMYFUiaACkqWCxKFgiiKzHI5cNLXTMsL6XpBY48vFsYeWxDjXifZp6jycua9x7xmujzyOuly+fVH3jMZkytICCgUxq0ru8/E54KStuU9T3jt8rZCZa+d+x6Z1zmWQ9wBYccMSenpoyK01ro2Y5WGc4JmaJjScDA0HOnzuPXSJOXDQWl4OLN9XPlwUCpNUl5eL+WXRyTLpYDh4WA4RuswUEraUS4bTvcpRVI2sjxMZp9kv9JwjNuXkeU6upL+iEhkgmbyoJoYWCBG15OwGQ2gkTJg4exZ/NN1F0173R0QZtMg+YZc9G9RcoyGUhIWpZGgyYRLZIIpEy7DmeOGc56HI3n93GfSkIskGIeHIcisB5nXmfw9yseMvPZwuaz8XuV98187vz5kXidy6jH62uXXCpJto6+XLEdAR2t1PsodEGZWVVJyqssfNsce/2LKzMxyOSDMzCyXA8LMzHI5IMzMLJcDwszMcjkgzMwslwPCzMxyOSDMzCxXXd1RTlIP8NwRHr4I2DmN1TkWuM2NwW1uDEfa5lMiojNvQ10FxEshqWuy2+7VK7e5MbjNjaEabfYpJjMzy+WAMDOzXA6IUbfWugI14DY3Bre5MUx7mz0GYWZmudyDMDOzXA4IMzPL1fABIekySU9J2iDpxlrXZ7pIWi1ph6T1mbIFkr4j6en0+bjMtg+lf4OnJP1ybWr90kg6SdK/SXpC0mOS/jAtr9t2S2qV9BNJj6Rt/mhaXrdtLpNUlPQzSd9M1+u6zZI2SVonaa2krrSsum2O9JZ4jfgAisAzwGlAC/AIsLLW9Zqmtr0OOA9Ynyn7c+DGdPlG4KZ0eWXa9lnAqenfpFjrNhxBm5cA56XLHcDP07bVbbtJbkk8J11uBv4TuLCe25xp+/uAO4Bvput13WZgE7BoXFlV29zoPYgLgA0RsTEiBoA7gStqXKdpERE/BF4YV3wFcFu6fBtwZab8zojoj4hngQ0kf5tjSkRsi4ifpsv7gSeApdRxuyNxIF1tTh9BHbcZQNIy4FeAz2eK67rNk6hqmxs9IJYCmzPr3WlZvTo+IrZB8mEKLE7L6+7vIGk5cC7JN+q6bnd6qmUtsAP4TkTUfZuBTwF/BAxnyuq9zQHcJ+lhSdemZVVtc6PfR1w5ZY143W9d/R0kzQH+H/DeiNgn5TUv2TWn7Jhrd0SUgFdLmg/cJemsw+x+zLdZ0q8COyLiYUmvr+SQnLJjqs2p10TEVkmLge9IevIw+05Lmxu9B9ENnJRZXwZsrVFdjobnJS0BSJ93pOV183eQ1EwSDl+JiK+lxXXfboCI2AN8H7iM+m7za4Bfl7SJ5LTwGyTdTn23mYjYmj7vAO4iOWVU1TY3ekA8BKyQdKqkFuBq4J4a16ma7gHekS6/A/h6pvxqSbMknQqsAH5Sg/q9JEq6Cl8AnoiIv85sqtt2S+pMew5IagMuBZ6kjtscER+KiGURsZzk/+z3IuJ3qOM2S5otqaO8DLwJWE+121zrkflaP4A3k1zt8gzwx7WuzzS266vANmCQ5NvEu4CFwP3A0+nzgsz+f5z+DZ4CLq91/Y+wzReTdKMfBdamjzfXc7uBc4CfpW1eD3wkLa/bNo9r/+sZvYqpbttMcqXlI+njsfJnVbXb7Kk2zMwsV6OfYjIzs0k4IMzMLJcDwszMcjkgzMwslwPCzMxyOSDMpkBSKZ1Ns/yYthmAJS3Pzr5rVmuNPtWG2VT1RsSra10Js6PBPQizaZDO1X9Tem+Gn0h6WVp+iqT7JT2aPp+clh8v6a70Pg6PSPrv6UsVJX0uvbfDfemvo81qwgFhNjVt404xvSWzbV9EXAD8Hclso6TLX4qIc4CvADen5TcDP4iIV5Hct+OxtHwF8OmIOBPYA1xV5faYTcq/pDabAkkHImJOTvkm4A0RsTGdMHB7RCyUtBNYEhGDafm2iFgkqQdYFhH9mddYTjJd94p0/YNAc0R8vPotM5vIPQiz6ROTLE+2T57+zHIJjxNaDTkgzKbPWzLPD6bL/0Ey4yjA24AH0uX7gd+HkRv+zD1alTSrlL+dmE1NW3r3trJ7I6J8qessSf9J8sXrrWnZe4DVkj4A9GabhskAAABTSURBVADvTMv/ELhV0rtIegq/TzL7rtmM4TEIs2mQjkGsioidta6L2XTxKSYzM8vlHoSZmeVyD8LMzHI5IMzMLJcDwszMcjkgzMwslwPCzMxy/X8rP6mFwsMMTwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAaj0lEQVR4nO3de5RdZZ3m8e+Tyj1BgRADpEIqYuhcDGagCFkKNGN6BBQ6YisXnUVgqYBDRLvVAYGWdhwcutWebhZgGhwaWINEEFHEDJfhog0ioYBAUgTIhQhluFSSZoCQC6n85o+9q7JP1amkktTOSdX7fNaqdfbZZ59zfu9Zq85z3ndfXkUEZmaWrgG1LsDMzGrLQWBmljgHgZlZ4hwEZmaJcxCYmSXOQWBmljgHgZlZ4hwE1q9JeljSv0saUsJrS9KFkpZIWi+pRdLtkqb19nuZlclBYP2WpAbgWCCAvyzhLf4Z+BpwIbA/cBjwS+BTO/tCkgb2bmlmPecgsP7sLOAPwI3AnOIDksZJ+oWkVklrJV1deOzLkpZKelvSc5KO6PzCkiYCFwBnRsSDEbEpIt6NiFsi4sp8m4clfanwnLMlPVK4H5IukLQMWCZpnqQfdnqfX0n6m3z5YEl35DW/JOnCXviMzBwE1q+dBdyS/50gaQyApDrgbuCPQAMwFpifP/Y54O/y576PrCextsprzwJaImLhbtb4aeBoYArwU+B0Scpr2Q/4BDBf0gDg18Azeb2zgK9LOmE339/MQWD9k6RjgPHAbRHxJLAC+Hz+8AzgYOBbEbE+IjZGRPsv9S8B/xART0RmeUT8scpbjAJe7YVS/0dErIuIDcC/kQ1jHZs/9lngsYhYDRwFjI6I/xYRmyNiJXA9cEYv1GCJcxBYfzUHuC8i1uT3f8q24aFxwB8jYkuV540jC40dWQsctNtVwivtC5FdAXI+cGa+6vNkvRnIQu1gSW+2/wGXAGN6oQZLnHdQWb8jaRhwGlAn6bV89RBgX0kfIfvyPUTSwCph8ApwaA/e5gHgGkmNEdHUzTbrgeGF+wdW2abz5X9vBe6TdCXZkNGphbpeioiJPajNbKe4R2D90aeBNrJx9+n532SyoZezgIVkwzpXShohaaikj+XP/QnwTUlH5oeHfkjS+M5vEBHLgGuBWyUdL2lw/jpnSLo432wR8BlJwyV9CPjijgqPiKeB1ryOeyPizfyhhcBbki6SNExSnaQPSzpqVz4gsyIHgfVHc4B/jYiXI+K19j/gauALgIBTgA8BLwMtwOkAEXE7cAXZUNLbZIeD7t/N+1yYv+Y1wJtkQ0qnku3UBfifwGbgdeAmtg3z7MitwF/kNZDX1ZbXPB14CVhDFhbv7+FrmnVLnpjGzCxt7hGYmSXOQWBmljgHgZlZ4hwEZmaJ63PnERxwwAHR0NBQ6zLMzPqUJ598ck1EjK72WJ8LgoaGBpqaujt/x8zMqpFU7VIpgIeGzMyS5yAwM0ucg8DMLHEOAjOzxDkIzMwSV1oQSLpB0huSlnTzuCRdJWm5pGerTQdoZmblK7NHcCNw4nYePwmYmP+dC/y4xFrMzKwbpZ1HEBG/k9SwnU1mAzfnszL9QdK+kg6KiN6Y/q9HfvPsq2zZupUVb7yzp97SzGyXNTbsz3GHVT0nbLfU8oSysRSm6SO7JvxYqswDK+lcsl4DhxxySK+8+RtvbeSCnz5VeI9eeVkzs9Kc/+eH9rsgqPbVW3VyhIi4DrgOoLGxsVcmUNiyddvLfOaIsfzjadN742XNzPqcWh411EI2UXi7emD1nnrzgQO25dDgOh88ZWbpquU34F3AWfnRQzOB/7cn9w+oMBY0sM7jQmaWrtKGhiTdChwPHCCpBbgcGAQQEfOABcAngeXAu8A5ZdVSTRRGoQa5R2BmCSvzqKEzd/B4ABeU9f47Upyq2UFgZilL9htwaxR7BB4aMrN0JRwE25YHDkj2YzAzSzgICkkweGCyH4OZWbpBEBU9Ag8NmVm60g0CHzVkZgYkHARbK44aco/AzNKVcBC4R2BmBgkHQRSCYKCDwMwSluw3oIeGzMwyCQeBh4bMzCDlINi6bdlBYGYpS/YbsHj4qK8+amYpSzcICvsIPB+BmaUs2W9A7yMwM8sk+w1YcdE5Dw2ZWcISDoLCRefcIzCzhCX7DVh5Qpl7BGaWroSDYNuy9xGYWcqS/QasOLPYE9OYWcKS/QasOGpooIeGzCxdDgKgzhPTmFnCkg2C4j4C4SAws3QlGwTFHsHwwXU1rMTMrLaSDYL2HPjxF45gxJCBtS3GzKyGkg2C9h7BmPcPrXElZma1lWwQtPcIBsj7B8wsbckGQXuPwAcMmVnqEg6C7NY9AjNLXcJBEDveyMwsAckGQXQMDblHYGZpSzgIsltfZsjMUpfs16D3EZiZZRIOAh81ZGYGJQeBpBMlvSBpuaSLqzy+n6Q7JT0raaGkD5dZT1F7EMg9AjNLXGlBIKkOuAY4CZgCnClpSqfNLgEWRcThwFnAP5dVT2c+oczMLFNmj2AGsDwiVkbEZmA+MLvTNlOABwAi4nmgQdKYEmsCYNOWNt5r2wrg646aWfLKvNraWOCVwv0W4OhO2zwDfAZ4RNIMYDxQD7xeYl382WX3dCy7R2BmqSuzR1DtG7bzWVxXAvtJWgR8FXga2NLlhaRzJTVJamptbe3dIp0DZpa4MnsELcC4wv16YHVxg4h4CzgHQNle25fyPzptdx1wHUBjY2OvnhI8wIcNmVniyuwRPAFMlDRB0mDgDOCu4gaS9s0fA/gS8Ls8HPYY54CZpa60HkFEbJE0F7gXqANuiIhmSefnj88DJgM3S2oDngO+WFY93fE+AjNLXalTc0XEAmBBp3XzCsuPARPLrGFHHANmlrpkzyxu5xPKzCx1yQeB9xGYWeocBO4RmFniHAQOAjNLXPJBoOQ/ATNLXfJfg+4PmFnqkg8CDw2ZWeocBA4CM0tc8kHgHDCz1CUfBO4RmFnqHATOATNLnIPAPQIzS1zyQeAcMLPUOQicBGaWuKSDwPsHzMySDwIngZmZg8DMLHFJB4FzwMzMQWBmlrykg8BDQ2ZmDoJal2BmVnNJB4FzwMwswSCIiFqXYGa2V0kwCKovm5mlKr0gKCxvdRKYmSUYBIUvfweBmVmKQVBY3uocMDNLMAgq9hE4CczM0gsCikNDNSzEzGwvkV4QuEdgZlYhuSAoco/AzCzBIHAnwMysUnpBgJPAzKwovSBwDpiZVSg1CCSdKOkFScslXVzl8fdL+rWkZyQ1SzqnzHoA9wfMzDopLQgk1QHXACcBU4AzJU3ptNkFwHMR8RHgeOBHkgaXVRP4SCEzs87K7BHMAJZHxMqI2AzMB2Z32iaAfSQJGAmsA7aUWJN7BGZmnZQZBGOBVwr3W/J1RVcDk4HVwGLgaxGxtfMLSTpXUpOkptbW1t0qqtgh+LtTOndQzMzSU2YQVJv2pfMP8hOARcDBwHTgaknv6/KkiOsiojEiGkePHr17VeUV/O3JUzj7YxN277XMzPqBMoOgBRhXuF9P9su/6BzgF5FZDrwETCqxpo7DRz05mZlZpswgeAKYKGlCvgP4DOCuTtu8DMwCkDQG+DNgZYk1dQwNeZpKM7PMwLJeOCK2SJoL3AvUATdERLOk8/PH5wHfA26UtJjsR/pFEbGmrJpg29iUc8DMLFNaEABExAJgQad18wrLq4FPlFlDlZoAkLsEZmZAimcW57fOATOzTHpB0L6PoLZlmJntNXYYBJJGSBpQuD9A0vByyypPx0Xn3CUwMwN61iN4ACh+8Q8H/m855ewB7hGYmVXoSRAMjYh32u/ky324R5Bxh8DMLNOTIFgv6Yj2O5KOBDaUV1K5tu0jcBKYmUHPDh/9OnC7pPazgg8CTi+vpHJ1nFnsHDAzA3oQBBHxhKRJZGf9Cng+It4rvbKStM9TPMBBYGYG9OyooQuAERGxJCIWAyMl/ZfySytHxwllHhoyMwN6to/gyxHxZvudiPh34MvllVSu8DUmzMwq9CQIBqhwPYZ85rFSZxHbE5wDZmaZnuwsvhe4TdI8sqMvzwf+T6lVlWjb1UcdBWZm0LMguAg4F/gK2Q/pp8mOHOqTPB+BmVmlHQ4N5VNH/oFsnoBGsvkDlpZcV2k8H4GZWaVuewSSDiObTOZMYC3wM4CI+I97prRy+MxiM7NK2xsaeh74N+CUfBpJJP31HqmqRD581Mys0vaGhv4KeA14SNL1kmbRD4bW3SMwM6vUbRBExJ0RcTrZZPIPA38NjJH0Y0l7dFax3tRxHoGZmQE921m8PiJuiYiTgXpgEXBx6ZWVxlNVmpkV7dQMZRGxLiL+JSI+XlZBZfMMZWZmldKbqjK/dYfAzCyTXhB4PgIzswrpBYHnIzAzq5BeEHgfgZlZhXSDwElgZgakGAR4QgIzs6L0gsA9AjOzCskFQTvngJlZJrkg8MQ0ZmaV0gsCT0xjZlYhvSDwPgIzswrpBUF+6yAwM8ukFwSemMbMrEKpQSDpREkvSFouqculqyV9S9Ki/G+JpDZJ+5dZU8d0BM4BMzOgxCCQVAdcA5wETAHOlDSluE1E/CAipkfEdODbwG8jYl1ZNWXvmddX5puYmfUhZfYIZgDLI2JlRGwG5gOzt7P9mcCtJdaT88Q0ZmZFZQbBWOCVwv2WfF0XkoYDJwJ3dPP4uZKaJDW1trbuVlHuEZiZVSozCKp913Y3Y/ApwKPdDQtFxHUR0RgRjaNHj96tonzUkJlZpTKDoAUYV7hfD6zuZtsz2CPDQp6YxsysszKD4AlgoqQJkgaTfdnf1XkjSe8H/hz4VYm1dOg4fNQ5YGYGwMCyXjgitkiaC9wL1AE3RESzpPPzx+flm54K3BcR68uqpaKu/NY5YGaWKS0IACJiAbCg07p5ne7fCNxYZh2V75cvOAnMzIAUzyzGZxabmRUlFwT4onNmZhWSCwKPDJmZVUovCDwxjZlZhfSCAB8+amZWlF4Q+BITZmYV0guC/NY9AjOzTHJB8OJrb+dLTgIzM0gsCFatWc8VC5YC7hGYmbVLKgjWvLOpY9k5YGaWSSoIinz4qJlZJt0gqHUBZmZ7iXSDwElgZgakHATuE5iZASkHgXPAzAxIOAjMzCyTbBC4R2Bmlkk3CLyPwMwMSDkInANmZkBiQRCFZQeBmVkmqSBo27otCjw0ZGaWSSoIthaDwDlgZgYkFgRtUewRmJkZpBYE7hGYmXWRVBBEcW+x+wRmZkBiQeAegZlZV2kFgfcRmJl1kVQQVB415CgwM4PUgqCwj8AxYGaWSSoIKoaGnARmZkBiQbDVZxabmXWRVBD4qCEzs66SCoKtlScSmJkZCQeBewRmZplSg0DSiZJekLRc0sXdbHO8pEWSmiX9tsx62rZWvG+Zb2Vm1mcMLOuFJdUB1wD/CWgBnpB0V0Q8V9hmX+Ba4MSIeFnSB8qqB3xCmZlZNWX2CGYAyyNiZURsBuYDsztt83ngFxHxMkBEvFFiPYSHhszMuigzCMYCrxTut+Trig4D9pP0sKQnJZ1V7YUknSupSVJTa2vrLhfkiWnMzLoqMwiqfdN2PmxnIHAk8CngBOBvJR3W5UkR10VEY0Q0jh49epcL8uGjZmZdlbaPgKwHMK5wvx5YXWWbNRGxHlgv6XfAR4AXyyhoq/cRmJl1UWaP4AlgoqQJkgYDZwB3ddrmV8CxkgZKGg4cDSwtq6CtFbPXl/UuZmZ9S2k9gojYImkucC9QB9wQEc2Szs8fnxcRSyXdAzwLbAV+EhFLyqrJ+wjMzLoqc2iIiFgALOi0bl6n+z8AflBmHe08eb2ZWVdJnVns8wjMzLpKKgg8MY2ZWVdpBYEnpjEz6yKpIPDENGZmXSUVBJ6Yxsysq1KPGtrbtHlsyGyv9N5779HS0sLGjRtrXUqfN3ToUOrr6xk0aFCPn5NUEFTkgIPAbK/R0tLCPvvsQ0NDgw/k2A0Rwdq1a2lpaWHChAk9fl5aQ0M+fNRsr7Rx40ZGjRrlENhNkhg1atRO96ySCoI2Hz5qttfy/2Tv2JXPMa0gcI/AzKyLpILAE9OYmXWVVBD4onNmZl0lFgTblt0jMLPuzJ07l/Hjx9e6jD0mqSAoHjVkZlbNSy+9xMMPP8zmzZt5++23S3uftra20l57ZyV2HsG2IBjgLoHZXum7v27mudVv9eprTjn4fVx+ytQebXv55Zdz2WWXcf3119Pc3MzMmTMBWL16NV/96ldZuXIlGzZs4Oabb6a+vr7LuhkzZjBz5kzmz59PQ0MDf/rTn5g9ezZNTU187nOfY9y4cTz99NPMmjWLSZMm8cMf/pANGzawzz77cOeddzJ69Oiq7zVs2DDOP/98Hn30UQCeeuopvvnNb/Lggw/u9ueTVBB4zmIz257m5maWLFnCTTfdxCOPPNIRBFu2bOGkk07iiiuu4OSTT+bdd9+lra2NY445psu6iODll1/uGFp69tlnmTZtGgCLFy9m8uTJPPTQQwCsXbuWz372swB897vf5bbbbuO8886r+l4jRoxgxYoVtLW1UVdXxze+8Q1+9KMf9Uq7kwoCn1Bmtvfr6S/3Mlx66aV873vfQxKTJ09myZJswsRf/vKXTJ48mZNPPhmA4cOH8/Of/7zLOoBly5YxYcKEjuP524Ng48aNrFu3ju985zsd73fjjTfys5/9jE2bNvHaa6/x/e9/v+p7tZs6dSrNzc0sW7aMQw45hCOOOKJX2p1UEPiEMjPrzuOPP869997LokWLuOCCC9i4cSOHH344AIsWLeoYImpXbR1kv/rbewAATU1NnHfeeTQ3N3P00UczcGD2tXvzzTezcOFCHnzwQUaOHMlxxx3H1KlTufvuu6u+LsDMmTN59NFHufbaa7nnnnt6q+mp7SzetuwYMLOiSy65hLvvvptVq1axatUqnnnmmY4ewYEHHkhzc3PHtq2trVXXAaxbt45hw4YBsHTpUn7zm98wbdo0Fi9e3BEskAXGRz/6UUaOHMkdd9zB73//e6ZNm9bt60IWBJdddhmnnnoqY8eO7bW2pxUE3kdgZlXcf//9bNq0iVmzZnWsGzNmDOvXr2fdunWcffbZvP7660ydOpXp06fz2GOPVV0HcMIJJ/DAAw9w2mmncfvttzNq1CjGjBnTJQjmzJnDVVddxbHHHsuLL77IBz/4QUaMGNHt6wJMmjSJIUOGcNFFF/Vq+xV97JDKxsbGaGpq2qXnnv2vC3n4hSxdV135qd4sy8x2w9KlS5k8eXKty9jrzZ07l6OOOoo5c+Zsd7tqn6ekJyOisdr2yfQIfvtia0cImJn1JStWrGDSpEls2LBhhyGwK5LZWTxyyEA+Oe1AGkaNYP8Rg2tdjplZjx166KE8//zzpb1+MkFw5Pj9OHL8kbUuw8xsr5PM0JCZmVXnIDCzvUJfO3Blb7Urn6ODwMxqbujQoaxdu9ZhsJva5yweOnToTj0vmX0EZrb3qq+vp6WlpeLkKds1Q4cOpb6+fqee4yAws5obNGgQEyZMqHUZyfLQkJlZ4hwEZmaJcxCYmSWuz11rSFIr8MddfPoBwJpeLKcvcJvT4DanYXfaPD4iRld7oM8Fwe6Q1NTdRZf6K7c5DW5zGspqs4eGzMwS5yAwM0tcakFwXa0LqAG3OQ1ucxpKaXNS+wjMzKyr1HoEZmbWiYPAzCxxyQSBpBMlvSBpuaSLa11Pb5F0g6Q3JC0prNtf0v2SluW3+xUe+3b+Gbwg6YTaVL17JI2T9JCkpZKaJX0tX99v2y1pqKSFkp7J2/zdfH2/bTOApDpJT0u6O7/fr9sLIGmVpMWSFklqyteV2+6I6Pd/QB2wAvggMBh4BphS67p6qW3HAUcASwrr/gG4OF++GPj7fHlK3vYhwIT8M6mrdRt2oc0HAUfky/sAL+Zt67ftBgSMzJcHAY8DM/tzm/N2/A3wU+Du/H6/bm/ellXAAZ3WldruVHoEM4DlEbEyIjYD84HZNa6pV0TE74B1nVbPBm7Kl28CPl1YPz8iNkXES8Byss+mT4mIVyPiqXz5bWApMJZ+3O7IvJPfHZT/Bf24zZLqgU8BPyms7rft3YFS251KEIwFXincb8nX9VdjIuJVyL40gQ/k6/vd5yCpAfgPZL+Q+3W782GSRcAbwP0R0d/b/E/AfwW2Ftb15/a2C+A+SU9KOjdfV2q7U5mPQFXWpXjcbL/6HCSNBO4Avh4Rb0nVmpdtWmVdn2t3RLQB0yXtC9wp6cPb2bxPt1nSycAbEfGkpON78pQq6/pMezv5WESslvQB4H5Jz29n215pdyo9ghZgXOF+PbC6RrXsCa9LOgggv30jX99vPgdJg8hC4JaI+EW+ut+3GyAi3gQeBk6k/7b5Y8BfSlpFNpT7cUn/m/7b3g4RsTq/fQO4k2yop9R2pxIETwATJU2QNBg4A7irxjWV6S5gTr48B/hVYf0ZkoZImgBMBBbWoL7douyn//8ClkbEPxYe6rftljQ67wkgaRjwF8Dz9NM2R8S3I6I+IhrI/l8fjIj/TD9tbztJIyTt074MfAJYQtntrvUe8j24J/6TZEeXrAAurXU9vdiuW4FXgffIfh18ERgFPAAsy2/3L2x/af4ZvACcVOv6d7HNx5B1f58FFuV/n+zP7QYOB57O27wE+E6+vt+2udCO49l21FC/bi/ZkY3P5H/N7d9VZbfbl5gwM0tcKkNDZmbWDQeBmVniHARmZolzEJiZJc5BYGaWOAeBWSeS2vIrP7b/9drVaiU1FK8Ua7Y3SOUSE2Y7Y0NETK91EWZ7insEZj2UXyf+7/N5ARZK+lC+frykByQ9m98ekq8fI+nOfA6BZyR9NH+pOknX5/MK3JefKWxWMw4Cs66GdRoaOr3w2FsRMQO4muzqmOTLN0fE4cAtwFX5+quA30bER8jmjGjO108EromIqcCbwF+V3B6z7fKZxWadSHonIkZWWb8K+HhErMwvevdaRIyStAY4KCLey9e/GhEHSGoF6iNiU+E1GsguIT0xv38RMCgi/nv5LTOrzj0Cs50T3Sx3t001mwrLbXhfndWYg8Bs55xeuH0sX/492RUyAb4APJIvPwB8BTomlXnfnirSbGf4l4hZV8PymcDa3RMR7YeQDpH0ONmPqDPzdRcCN0j6FtAKnJOv/xpwnaQvkv3y/wrZlWLN9ireR2DWQ/k+gsaIWFPrWsx6k4eGzMwS5x6BmVni3CMwM0ucg8DMLHEOAjOzxDkIzMwS5yAwM0vc/wd6zAVCElemGwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 利用鸢尾花数据集，实现前向传播、反向传播，可视化loss曲线\n",
    "\n",
    "# 导入所需模块\n",
    "import tensorflow as tf\n",
    "from sklearn import datasets\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import time  ##1##\n",
    "\n",
    "# 导入数据，分别为输入特征和标签\n",
    "x_data = datasets.load_iris().data\n",
    "y_data = datasets.load_iris().target\n",
    "\n",
    "# 随机打乱数据（因为原始数据是顺序的，顺序不打乱会影响准确率）\n",
    "# seed: 随机数种子，是一个整数，当设置之后，每次生成的随机数都一样（为方便教学，以保每位同学结果一致）\n",
    "np.random.seed(116)  # 使用相同的seed，保证输入特征和标签一一对应\n",
    "np.random.shuffle(x_data)\n",
    "np.random.seed(116)\n",
    "np.random.shuffle(y_data)\n",
    "tf.random.set_seed(116)\n",
    "\n",
    "# 将打乱后的数据集分割为训练集和测试集，训练集为前120行，测试集为后30行\n",
    "x_train = x_data[:-30]\n",
    "y_train = y_data[:-30]\n",
    "x_test = x_data[-30:]\n",
    "y_test = y_data[-30:]\n",
    "\n",
    "# 转换x的数据类型，否则后面矩阵相乘时会因数据类型不一致报错\n",
    "x_train = tf.cast(x_train, tf.float32)\n",
    "x_test = tf.cast(x_test, tf.float32)\n",
    "\n",
    "# from_tensor_slices函数使输入特征和标签值一一对应。（把数据集分批次，每个批次batch组数据）\n",
    "train_db = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(32)\n",
    "test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)\n",
    "\n",
    "# 生成神经网络的参数，4个输入特征故，输入层为4个输入节点；因为3分类，故输出层为3个神经元\n",
    "# 用tf.Variable()标记参数可训练\n",
    "# 使用seed使每次生成的随机数相同（方便教学，使大家结果都一致，在现实使用时不写seed）\n",
    "w1 = tf.Variable(tf.random.truncated_normal([4, 3], stddev=0.1, seed=1))\n",
    "b1 = tf.Variable(tf.random.truncated_normal([3], stddev=0.1, seed=1))\n",
    "\n",
    "lr = 0.1  # 学习率为0.1\n",
    "train_loss_results = []  # 将每轮的loss记录在此列表中，为后续画loss曲线提供数据\n",
    "test_acc = []  # 将每轮的acc记录在此列表中，为后续画acc曲线提供数据\n",
    "epoch = 500  # 循环500轮\n",
    "loss_all = 0  # 每轮分4个step，loss_all记录四个step生成的4个loss的和\n",
    "\n",
    "##########################################################################\n",
    "m_w, m_b = 0, 0\n",
    "v_w, v_b = 0, 0\n",
    "beta1, beta2 = 0.9, 0.999\n",
    "delta_w, delta_b = 0, 0\n",
    "global_step = 0\n",
    "##########################################################################\n",
    "\n",
    "# 训练部分\n",
    "now_time = time.time()  ##2##\n",
    "for epoch in range(epoch):  # 数据集级别的循环，每个epoch循环一次数据集\n",
    "    for step, (x_train, y_train) in enumerate(train_db):  # batch级别的循环 ，每个step循环一个batch\n",
    " ##########################################################################       \n",
    "        global_step += 1\n",
    " ##########################################################################       \n",
    "        with tf.GradientTape() as tape:  # with结构记录梯度信息\n",
    "            y = tf.matmul(x_train, w1) + b1  # 神经网络乘加运算\n",
    "            y = tf.nn.softmax(y)  # 使输出y符合概率分布（此操作后与独热码同量级，可相减求loss）\n",
    "            y_ = tf.one_hot(y_train, depth=3)  # 将标签值转换为独热码格式，方便计算loss和accuracy\n",
    "            loss = tf.reduce_mean(tf.square(y_ - y))  # 采用均方误差损失函数mse = mean(sum(y-out)^2)\n",
    "            loss_all += loss.numpy()  # 将每个step计算出的loss累加，为后续求loss平均值提供数据，这样计算的loss更准确\n",
    "        # 计算loss对各个参数的梯度\n",
    "        grads = tape.gradient(loss, [w1, b1])\n",
    "\n",
    "##########################################################################\n",
    " # adam\n",
    "        m_w = beta1 * m_w + (1 - beta1) * grads[0]\n",
    "        m_b = beta1 * m_b + (1 - beta1) * grads[1]\n",
    "        v_w = beta2 * v_w + (1 - beta2) * tf.square(grads[0])\n",
    "        v_b = beta2 * v_b + (1 - beta2) * tf.square(grads[1])\n",
    "\n",
    "        m_w_correction = m_w / (1 - tf.pow(beta1, int(global_step)))\n",
    "        m_b_correction = m_b / (1 - tf.pow(beta1, int(global_step)))\n",
    "        v_w_correction = v_w / (1 - tf.pow(beta2, int(global_step)))\n",
    "        v_b_correction = v_b / (1 - tf.pow(beta2, int(global_step)))\n",
    "\n",
    "        w1.assign_sub(lr * m_w_correction / tf.sqrt(v_w_correction))\n",
    "        b1.assign_sub(lr * m_b_correction / tf.sqrt(v_b_correction))\n",
    "##########################################################################\n",
    "\n",
    "    # 每个epoch，打印loss信息\n",
    "    print(\"Epoch {}, loss: {}\".format(epoch, loss_all / 4))\n",
    "    train_loss_results.append(loss_all / 4)  # 将4个step的loss求平均记录在此变量中\n",
    "    loss_all = 0  # loss_all归零，为记录下一个epoch的loss做准备\n",
    "\n",
    "    # 测试部分\n",
    "    # total_correct为预测对的样本个数, total_number为测试的总样本数，将这两个变量都初始化为0\n",
    "    total_correct, total_number = 0, 0\n",
    "    for x_test, y_test in test_db:\n",
    "        # 使用更新后的参数进行预测\n",
    "        y = tf.matmul(x_test, w1) + b1\n",
    "        y = tf.nn.softmax(y)\n",
    "        pred = tf.argmax(y, axis=1)  # 返回y中最大值的索引，即预测的分类\n",
    "        # 将pred转换为y_test的数据类型\n",
    "        pred = tf.cast(pred, dtype=y_test.dtype)\n",
    "        # 若分类正确，则correct=1，否则为0，将bool型的结果转换为int型\n",
    "        correct = tf.cast(tf.equal(pred, y_test), dtype=tf.int32)\n",
    "        # 将每个batch的correct数加起来\n",
    "        correct = tf.reduce_sum(correct)\n",
    "        # 将所有batch中的correct数加起来\n",
    "        total_correct += int(correct)\n",
    "        # total_number为测试的总样本数，也就是x_test的行数，shape[0]返回变量的行数\n",
    "        total_number += x_test.shape[0]\n",
    "    # 总的准确率等于total_correct/total_number\n",
    "    acc = total_correct / total_number\n",
    "    test_acc.append(acc)\n",
    "    print(\"Test_acc:\", acc)\n",
    "    print(\"--------------------------\")\n",
    "total_time = time.time() - now_time  ##3##\n",
    "print(\"total_time\", total_time)  ##4##\n",
    "\n",
    "# 绘制 loss 曲线\n",
    "plt.title('Loss Function Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Loss')  # y轴变量名称\n",
    "plt.plot(train_loss_results, label=\"$Loss$\")  # 逐点画出trian_loss_results值并连线，连线图标是Loss\n",
    "plt.legend()  # 画出曲线图标\n",
    "plt.show()  # 画出图像\n",
    "\n",
    "# 绘制 Accuracy 曲线\n",
    "plt.title('Acc Curve')  # 图片标题\n",
    "plt.xlabel('Epoch')  # x轴变量名称\n",
    "plt.ylabel('Acc')  # y轴变量名称\n",
    "plt.plot(test_acc, label=\"$Accuracy$\")  # 逐点画出test_acc值并连线，连线图标是Accuracy\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# 请将loss曲线、ACC曲线、total_time记录到 class2\\优化器对比.docx  对比各优化器收敛情况\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
